You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/11/25 21:29:00 UTC

[07/19] hbase git commit: HBASE-12476 HydraBase consensus protocol

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/LocalConsensusServer.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/LocalConsensusServer.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/LocalConsensusServer.java
new file mode 100644
index 0000000..3de7ab8
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/LocalConsensusServer.java
@@ -0,0 +1,354 @@
+package org.apache.hadoop.hbase.consensus.server;
+
+import io.airlift.units.Duration;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.consensus.client.QuorumClient;
+import org.apache.hadoop.hbase.consensus.quorum.AggregateTimer;
+import org.apache.hadoop.hbase.consensus.quorum.QuorumInfo;
+import org.apache.hadoop.hbase.consensus.quorum.RaftQuorumContext;
+import org.apache.hadoop.hbase.consensus.util.RaftUtil;
+import org.apache.hadoop.hbase.util.DaemonThreadFactory;
+import org.apache.hadoop.hbase.util.InfoServer;
+import org.apache.hadoop.hbase.util.serial.AsyncSerialExecutorServiceImpl;
+import org.apache.hadoop.hbase.util.serial.SerialExecutorService;
+import org.apache.log4j.Level;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.jboss.netty.logging.InternalLoggerFactory;
+import org.jboss.netty.logging.Slf4JLoggerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.weakref.jmx.MBeanExporter;
+
+import com.facebook.nifty.codec.DefaultThriftFrameCodecFactory;
+import com.facebook.nifty.codec.ThriftFrameCodecFactory;
+import com.facebook.nifty.core.NiftyTimer;
+import com.facebook.nifty.duplex.TDuplexProtocolFactory;
+import com.facebook.swift.codec.ThriftCodecManager;
+import com.facebook.swift.service.ThriftEventHandler;
+import com.facebook.swift.service.ThriftServer;
+import com.facebook.swift.service.ThriftServerConfig;
+import com.facebook.swift.service.ThriftServiceProcessor;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+
+public class LocalConsensusServer {
+  public static final ImmutableMap<String,TDuplexProtocolFactory>
+    SWIFT_PROTOCOL_FACTORIES = ImmutableMap.of(
+    "binary", TDuplexProtocolFactory.fromSingleFactory(
+      new TBinaryProtocol.Factory()),
+    "compact", TDuplexProtocolFactory.fromSingleFactory(
+      new TCompactProtocol.Factory())
+  );
+
+  public static final ImmutableMap<String,ThriftFrameCodecFactory>
+    SWIFT_FRAME_CODEC_FACTORIES = ImmutableMap.<String, ThriftFrameCodecFactory>of(
+    "buffered", new DefaultThriftFrameCodecFactory(),
+    "framed", new DefaultThriftFrameCodecFactory()
+  );
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+    LocalConsensusServer.class);
+  private final Configuration conf;
+  private ThriftServer server;
+  private final ConsensusService handler;
+  private final List<ThriftEventHandler> eventHandlers;
+
+  private InfoServer infoServer;
+
+  private ThriftServerConfig config;
+
+  private ExecutorService execServiceForThriftClients;
+
+  public final AggregateTimer aggregateTimer = new AggregateTimer();
+  public SerialExecutorService serialExecutorService;
+
+  @Inject
+  public LocalConsensusServer(final ConsensusService handler,
+                              final List<ThriftEventHandler> eventHandlers,
+                              final Configuration conf) {
+    this.eventHandlers = eventHandlers;
+    this.handler = handler;
+    this.conf = conf;
+
+    this.execServiceForThriftClients =
+        Executors.newFixedThreadPool(
+          HConstants.DEFAULT_QUORUM_CLIENT_NUM_WORKERS,
+          new DaemonThreadFactory("QuorumClient-"));
+  }
+
+  public void initialize(final ThriftServerConfig config) {
+    if (!(InternalLoggerFactory.getDefaultFactory()
+            instanceof Slf4JLoggerFactory)) {
+      InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory());
+    }
+
+    // Set the transport protocol
+    config.setProtocolName(conf.get(HConstants.CONSENSUS_TRANSPORT_PROTOCOL_KEY,
+      HConstants.CONSENSUS_TRANSPORT_PROTOCOL_DEFAULT));
+
+    // Set the codec factory
+    config.setTransportName(conf.get(HConstants.CONSENSUS_TRANSPORT_CODEC_KEY,
+      HConstants.CONSENSUS_TRANSPORT_CODEC_DEFAULT));
+
+    // "ConsensusAsyncSerialExecutor"
+    int numThreads = conf.getInt(HConstants.FSM_MUX_THREADPOOL_SIZE_KEY,
+      HConstants.DEFAULT_FSM_MUX_THREADPOOL_SIZE);
+    serialExecutorService =
+        new AsyncSerialExecutorServiceImpl(numThreads, "AsyncSerialExecutor");
+
+    ThriftServer newServer = new ThriftServer(
+      new ThriftServiceProcessor(
+        new ThriftCodecManager(), eventHandlers, handler),
+      config,
+      new NiftyTimer("thrift"),
+      SWIFT_FRAME_CODEC_FACTORIES,
+      SWIFT_PROTOCOL_FACTORIES,
+      ThriftServer.DEFAULT_WORKER_EXECUTORS,
+      ThriftServer.DEFAULT_SECURITY_FACTORY);
+
+    LOG.info("Start the thrift server with a unbounded queue !");
+
+    synchronized (this) {
+      server = newServer;
+      this.config = config;
+    }
+  }
+
+  public void startService() {
+    LOG.info("Starting the consensus service at port : " + config.getPort());
+    server.start();
+    LOG.info("Started the consensus service at port : " + config.getPort());
+  }
+
+  public void restartService() {
+    LOG.info("Restarting the consensus service at port : " + config.getPort());
+    getThriftServer().close();
+    initialize(this.config);
+    getThriftServer().start();
+    LOG.info("Restarted the consensus service at port: " + config.getPort());
+  }
+
+  public void stopService() {
+    LOG.info("Stopping the consensus service at port : " + config.getPort());
+    LOG.info(
+      com.google.common.base.Throwables.getStackTraceAsString(new Throwable()));
+    handler.stopService();
+    server.close();
+    execServiceForThriftClients.shutdownNow();
+  }
+
+  public ConsensusService getHandler() {
+    return handler;
+  }
+
+  public synchronized ThriftServer getThriftServer() {
+    return server;
+  }
+
+  public void startInfoServer(HServerAddress local, Configuration conf) throws IOException {
+    this.infoServer = new InfoServer(HConstants.QUORUM_MONITORING_PAGE_KEY,
+      local.getBindAddress(), server.getPort() + 1, false,
+                                     conf);
+    this.infoServer.setAttribute(HConstants.QUORUM_MONITORING_PAGE_KEY, this);
+    this.infoServer.start();
+  }
+
+  public void shutdownInfoServer() throws Exception {
+    this.infoServer.stop();
+  }
+
+  public QuorumClient getQuorumClient(QuorumInfo quorumInfo) throws IOException {
+    return new QuorumClient(quorumInfo, this.conf, execServiceForThriftClients);
+  }
+
+  @Override
+  public String toString() {
+    return "LocalConsensusServer@" + config.getPort()
+      + "[" + (server.isRunning() ? "RUNNING" : "OFFLINE") + "@" + server.getPort() + "]";
+  }
+
+  public static void main(String[] args) {
+    LOG.info("Starting Consensus service server");
+    final Configuration configuration = HBaseConfiguration.create();
+
+    Options opt = new Options();
+    opt.addOption("region", true, "the region id for the load test");
+    opt.addOption("servers", true, "A list of quorum server delimited by ,");
+    opt.addOption("localIndex", true, "The index of the local quorum server from the server list");
+    opt.addOption("debug", false, "Set the log4j as debug level." );
+    opt.addOption("jetty", false, "Enable the jetty server" );
+
+    String regionId = null;
+    String serverList;
+    String servers[] = null;
+    HServerAddress localHost = null;
+    int localIndex = -1;
+    HashMap<HServerAddress, Integer> peers = new HashMap<>();
+    int rank = 5;
+    boolean enableJetty = false;
+
+    try {
+      CommandLine cmd = new GnuParser().parse(opt, args);
+
+      if (cmd.hasOption("region")) {
+        regionId = cmd.getOptionValue("region");
+      }
+
+      if (cmd.hasOption("servers")) {
+        serverList = cmd.getOptionValue("servers");
+        servers = serverList.split(",");
+      }
+
+      if (cmd.hasOption("localIndex")) {
+        localIndex = Integer.parseInt(cmd.getOptionValue("localIndex"));
+      }
+
+      if (!cmd.hasOption("debug")) {
+        org.apache.log4j.Logger.getLogger(
+                "org.apache.hadoop.hbase.consensus").setLevel(Level.INFO);
+      }
+
+
+      if (cmd.hasOption("jetty")) {
+        enableJetty = true;
+      }
+
+      if (regionId == null || regionId.isEmpty() || servers == null || servers.length == 0) {
+        LOG.error("Wrong args !");
+        printHelp(opt);
+        System.exit(-1);
+      }
+
+      System.out.println("*******************");
+      System.out.println("region: " + regionId);
+      String hostname = null;
+
+      try {
+        hostname = InetAddress.getLocalHost().getHostName();
+        System.out.println("Current host name is " + hostname);
+      } catch (UnknownHostException e) {
+        System.out.println("Not able to retrieve the local host name");
+        System.exit(-1);
+      }
+
+      for (int i = 0; i < servers.length; i++) {
+        String localConsensusServerAddress = servers[i];
+
+        HServerAddress cur = new HServerAddress(localConsensusServerAddress);
+        peers.put(RaftUtil.getHRegionServerAddress(cur), rank--);
+        System.out.println("Quorum server " + localConsensusServerAddress);
+
+        if (localIndex != -1) { // local mode
+          if (localIndex == i) {
+            localHost = cur;
+          }
+        } else if (localConsensusServerAddress.contains(hostname)) { // remote mode
+          localHost = cur;
+        }
+      }
+      if (localHost == null) {
+        System.out.println("Error: no local server");
+        printHelp(opt);
+        System.exit(-1);
+      }
+      System.out.println("Local Quorum server " + localHost);
+      System.out.println("*******************");
+    } catch (Exception e) {
+      e.printStackTrace();
+      printHelp(opt);
+      System.exit(-1);
+    }
+
+    if (localIndex != -1) {
+      // overwrite the log directory for the local mode
+      configuration.set("hbase.consensus.log.path",  "/tmp/wal/" + localHost.getHostAddressWithPort());
+    }
+
+    // Start the local consensus server
+    ThriftServerConfig config =  new ThriftServerConfig().
+      setWorkerThreads(configuration.getInt(
+        HConstants.CONSENSUS_SERVER_WORKER_THREAD,
+        HConstants.DEFAULT_CONSENSUS_SERVER_WORKER_THREAD)).
+      setIoThreadCount(configuration.getInt(
+        HConstants.CONSENSUS_SERVER_IO_THREAD,
+        HConstants.DEFAULT_CONSENSUS_SERVER_IO_THREAD)).
+      setPort(localHost.getPort()).
+      setIdleConnectionTimeout(new Duration(1, TimeUnit.DAYS));
+
+    // Set the max progress timeout as 30 sec;
+    // The actual progress timeout will be the max progress timeout/rank
+    // And the heartbeat timeout will be the the max timeout/20;
+    configuration.setInt(HConstants.PROGRESS_TIMEOUT_INTERVAL_KEY, 30 * 1000);
+
+    MBeanExporter mbeanExporter = MBeanExporter.withPlatformMBeanServer();
+
+    LocalConsensusServer consensusServer = new LocalConsensusServer(
+      ConsensusServiceImpl.createConsensusServiceImpl(),
+      new ArrayList<ThriftEventHandler>(),
+      configuration);
+    consensusServer.initialize(config);
+    consensusServer.startService();
+    LOG.info("Started Consensus Service with port:" + config.getPort());
+
+    if (enableJetty) {
+      try {
+        consensusServer.startInfoServer(localHost, configuration);
+        System.out.println("Start the jetty server at " +
+          localHost.getBindAddress() + ":" + localHost.getPort() + 1);
+      } catch (Exception e) {
+        LOG.error("Unable to start the jetty server ", e);
+      }
+    }
+
+    // Set the region with the peers
+    HRegionInfo regionInfo = RaftUtil.createDummyRegionInfo(regionId, peers);
+
+    // Create the RaftQuorumContext
+    RaftQuorumContext context = new RaftQuorumContext(regionInfo.getQuorumInfo(),
+      configuration, localHost,
+      (regionInfo.getTableDesc().getNameAsString() + "."),
+      consensusServer.aggregateTimer,
+      consensusServer.serialExecutorService,
+      consensusServer.execServiceForThriftClients
+    );
+    context.getConsensusMetrics().export(mbeanExporter);
+
+    // Initialize the raft context
+    context.initializeAll(HConstants.UNDEFINED_TERM_INDEX);
+
+    // Register the raft context into the consensus server
+    consensusServer.getHandler().addRaftQuorumContext(context);
+    LOG.info("Registered the region " + regionId + " with the consensus server");
+  }
+
+  private static void printHelp(Options opt) {
+    new HelpFormatter().printHelp(
+      "QuorumLoadTestClient -region regionID -servers h1:port,h2:port," +
+        "h3:port...] [-localIndex index]", opt);
+  }
+
+  public ExecutorService getExecServiceForThriftClients() {
+    return execServiceForThriftClients;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/AbstractPeer.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/AbstractPeer.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/AbstractPeer.java
new file mode 100644
index 0000000..23bc9d6
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/AbstractPeer.java
@@ -0,0 +1,447 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.fsm.FiniteStateMachine;
+import org.apache.hadoop.hbase.consensus.fsm.FiniteStateMachineService;
+import org.apache.hadoop.hbase.consensus.fsm.FiniteStateMachineServiceImpl;
+import org.apache.hadoop.hbase.consensus.fsm.Util;
+import org.apache.hadoop.hbase.consensus.log.TransactionLogManager;
+import org.apache.hadoop.hbase.consensus.metrics.PeerMetrics;
+import org.apache.hadoop.hbase.consensus.protocol.EditId;
+import org.apache.hadoop.hbase.consensus.quorum.MutableRaftContext;
+import org.apache.hadoop.hbase.consensus.quorum.RaftQuorumContext;
+import org.apache.hadoop.hbase.consensus.raft.events.AppendResponseEvent;
+import org.apache.hadoop.hbase.consensus.raft.events.VoteResponseEvent;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+import org.apache.hadoop.hbase.consensus.rpc.AppendResponse;
+import org.apache.hadoop.hbase.consensus.rpc.VoteRequest;
+import org.apache.hadoop.hbase.consensus.rpc.VoteResponse;
+import org.apache.hadoop.hbase.consensus.server.ConsensusService;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendRequestEvent;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendResponseEvent;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerServerEventType;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerVoteRequestEvent;
+import org.apache.hadoop.hbase.consensus.util.RaftUtil;
+import org.apache.hadoop.hbase.ipc.thrift.exceptions.ThriftHBaseException;
+import org.apache.hadoop.hbase.regionserver.RaftEventListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.weakref.jmx.JmxException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+@NotThreadSafe
+public abstract class AbstractPeer implements PeerServer {
+  public int rank;
+
+  protected MutableRaftContext raftContext;
+  protected FiniteStateMachineService stateMachine;
+  protected int connectionRetry = 1;
+  protected int connectionRetryInterval;
+
+  private static Logger LOG = LoggerFactory.getLogger(AbstractPeer.class);
+  private final HServerAddress address;
+  private String sourceString = null;
+  /** Last successfully acked Edit Id by the peer */
+  private EditId lastEditID = TransactionLogManager.UNDEFINED_EDIT_ID;
+  private AppendRequest latestRequest;
+  private volatile boolean isStopped = false;
+  protected RaftEventListener dataStoreEventListener;
+  private final static long DEFAULT_ASYNC_SEND_OUTLIER = 50 * 1000 * 1000; // 50ms
+  private boolean useAggregateTimer;
+  private long timeLastRequestWasSent = 0;
+  private boolean useMuxForPSM;
+
+  private final PeerMetrics metrics;
+
+  public AbstractPeer(final HServerAddress address, final int rank,
+                      final MutableRaftContext replica,
+                      final Configuration conf) {
+    this.rank = rank;
+    this.address = address;
+    this.raftContext = replica;
+    connectionRetry = conf.getInt(
+      HConstants.RAFT_PEERSERVER_CONNECTION_RETRY_CNT, 1);
+    connectionRetryInterval = conf.getInt(
+      HConstants.RAFT_PEERSERVER_CONNECTION_RETRY_INTERVAL, 1000);
+
+    // MBean name properties can not contain colons, so replace with a period.
+    // TODO (arjen): Make sure this works for IPv6.
+    String peerId = address.getHostNameWithPort().replace(':', '.');
+    metrics = replica.getConsensusMetrics().createPeerMetrics(peerId);
+    useAggregateTimer = conf.getBoolean(
+      HConstants.QUORUM_USE_AGGREGATE_TIMER_KEY,
+      HConstants.QUORUM_USE_AGGREGATE_TIMER_DEFAULT);
+    useMuxForPSM = conf.getBoolean(HConstants.USE_FSMMUX_FOR_PSM,
+      HConstants.USE_FSMMUX_FOR_PSM_DEFAULT);
+  }
+
+  public HServerAddress getAddress() {
+    return address;
+  }
+
+  @Override
+  public int getRank() {
+    return rank;
+  }
+
+  @Override
+  public void setRank(int rank) {
+    this.rank = rank;
+  }
+
+  @Override
+  public AppendRequest getLatestRequest() {
+    return latestRequest;
+  }
+
+  @Override
+  public void setLatestRequest(final AppendRequest latestRequest) {
+    this.latestRequest = latestRequest;
+  }
+
+  protected MutableRaftContext getRaftContext() {
+    return raftContext;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return raftContext.getConf();
+  }
+
+  @Override
+  public void registerDataStoreEventListener(RaftEventListener listener) {
+    this.dataStoreEventListener = listener;
+  }
+
+  @Override
+  public void updatePeerAvailabilityStatus(boolean isAvailable) {
+    if (this.dataStoreEventListener != null) {
+      this.dataStoreEventListener.updatePeerAvailabilityStatus(
+        this.address.getHostAddressWithPort(), isAvailable);
+    } else {
+      LOG.warn("dataStoreEventListener has not been registered for this peer: " + toString());
+    }
+  }
+
+  public FiniteStateMachineService getStateMachineService() {
+    return stateMachine;
+  }
+
+  abstract protected ConsensusService getConsensusServiceAgent();
+  abstract protected void resetConnection()
+    throws ExecutionException, InterruptedException, TimeoutException;
+  abstract protected boolean clearConsensusServiceAgent(ConsensusService localAgent);
+
+  public void setStateMachineService(final FiniteStateMachineService fsm) {
+    stateMachine = fsm;
+  }
+
+  public void initializeStateMachine() {
+    if (stateMachine == null) {
+      FiniteStateMachine fsm = new PeerStateMachine(
+        "PeerFSM-" + raftContext.getQuorumName() + ":" +
+          raftContext.getRanking() + "-->" +
+          this.address.getHostAddressWithPort() +  ":" + this.rank,
+        this);
+
+      FiniteStateMachineService fsmService;
+      if (useMuxForPSM) {
+        fsmService = ((RaftQuorumContext) raftContext)
+          .createFiniteStateMachineService(fsm);
+      } else {
+        fsmService = new FiniteStateMachineServiceImpl(fsm);
+      }
+      setStateMachineService(fsmService);
+    }
+    stateMachine.offer(new Event(PeerServerEventType.START));
+  }
+
+  @Override
+  public void initialize() {
+    isStopped = false;
+    lastEditID = raftContext.getLastLogIndex();
+    try {
+      raftContext.getConsensusMetrics().exportPeerMetrics(metrics);
+    } catch (JmxException e) {
+      LOG.warn(String.format("Could not export metrics MBean: %s, reason: %s",
+              metrics, e.getReason()));
+    }
+    initializeStateMachine();
+  }
+
+  @Override
+  public void resetPeerContext() {
+    lastEditID = raftContext.getLastLogIndex();
+    latestRequest = null;
+  }
+
+  @Override
+  public void sendAppendEntries(AppendRequest request) {
+    // Append the event to the state machine
+    stateMachine.offer(new PeerAppendRequestEvent(request));
+  }
+
+  @Override
+  public void sendRequestVote(VoteRequest request) {
+    // Append the event to the state machine
+    stateMachine.offer(new PeerVoteRequestEvent(request));
+  }
+
+  @Override
+  public String getPeerServerName() {
+    return address.getHostAddressWithPort();
+  }
+
+  @Override
+  public String toString() {
+    return "[PeerServer: " + this.address + ", rank:" + this.rank + "]";
+  }
+
+  @Override
+  public MutableRaftContext getQuorumContext() {
+    return raftContext;
+  }
+
+  @Override
+  public EditId getLastEditID() {
+    return lastEditID;
+  }
+
+  @Override
+  public void enqueueEvent(final Event e) {
+    stateMachine.offer(e);
+  }
+
+  @Override
+  public void setLastEditID(final EditId lastEditID) {
+    this.lastEditID = lastEditID;
+  }
+
+  @Override
+  public void stop() {
+    isStopped = true;
+    stateMachine.offer(new Event(PeerServerEventType.HALT));
+    stateMachine.shutdown();
+    if (!Util.awaitTermination(
+        stateMachine, 3, 3, TimeUnit.SECONDS)) {
+      LOG.error("State Machine Service " + stateMachine.getName() +
+              " did not shutdown");
+    }
+    try {
+      raftContext.getConsensusMetrics().unexportPeerMetrics(metrics);
+    } catch (JmxException e) {
+      LOG.warn(String.format(
+                      "Could not un-export metrics MBean: %s, reason: %s",
+                      metrics, e.getReason()));
+    }
+  }
+
+  private String getTargetString() {
+    return this.toString();
+  }
+
+  private String getSourceString() {
+    // we don't have to synchronize it
+    if (sourceString == null) {
+      if (getRaftContext() != null) {
+        sourceString = getRaftContext().getMyAddress();
+        return sourceString;
+      } else {
+        return "?:?";
+      }
+    } else {
+      return sourceString;
+    }
+  }
+
+  @Override
+  public void sendAppendRequestWithCallBack(final AppendRequest request) {
+    if (isStopped) {
+      if (LOG.isTraceEnabled() || request.isTraceable()) {
+        LOG.debug("[AppendRequest Trace] " + getSourceString() + " not sending " + request + " to "
+          + getTargetString() + "; because we are stopped!");
+      }
+      return;
+    }
+
+    final ConsensusService localAgent = getConsensusServiceAgent();
+    if (localAgent == null) {
+      LOG.warn("Failed to get localAgent for " + this + " for the request " + request);
+      triggerRPCErrorEvent();
+      return;
+    }
+
+    assert request.validateFields();
+    assert request.getLeaderId() != null;
+
+    if (LOG.isTraceEnabled() || request.isTraceable()) {
+      LOG.debug("[AppendRequest Trace] " + getSourceString() + ": sending " + request + " to " +
+        getTargetString());
+    }
+    final long start = System.nanoTime();
+
+    long time = System.currentTimeMillis();
+
+    if (time - timeLastRequestWasSent > 1000) {
+      LOG.warn(getSourceString() + "-->" +
+        getTargetString() + " did not send append request in last " +
+        (time - timeLastRequestWasSent) + " ms . Current req " + request);
+    }
+    timeLastRequestWasSent = time;
+
+    RaftUtil.getThriftClientManager().getNiftyChannel(localAgent).executeInIoThread(
+      new Runnable() {
+        @Override public void run() {
+          try {
+
+            ListenableFuture<AppendResponse> futureResponse =
+              localAgent.appendEntries(request);
+            Futures.addCallback(futureResponse, new FutureCallback<AppendResponse>() {
+              @Override
+              public void onSuccess(AppendResponse response) {
+                getQuorumContext().offerEvent(new AppendResponseEvent(response));
+                metrics.getAppendEntriesLatency().add(System.nanoTime() - start,
+                  TimeUnit.NANOSECONDS);
+                if (LOG.isTraceEnabled() || request.isTraceable()) {
+                  LOG.debug("[AppendRequest Trace] Received the response: " + response);
+                }
+                getStateMachineService().offer(new PeerAppendResponseEvent(response));
+              }
+
+              @Override
+              public void onFailure(Throwable e) {
+                handleFailure(localAgent, request, e);
+              }
+            });
+          } catch (Throwable t) {
+            LOG.error("Could not send async request", t);
+            handleFailure(localAgent, request, t);
+          }
+        }
+      });
+
+  }
+
+  private void handleFailure(ConsensusService localAgent, AppendRequest request, Throwable e) {
+    if (LOG.isDebugEnabled()) {
+      if (e instanceof ThriftHBaseException) {
+        e = ((ThriftHBaseException)e).getServerJavaException();
+      }
+
+      LOG.debug(request + " (" + getSourceString() + "-->" +
+          getTargetString() + ") FAILED due to ", e);
+    }
+    if (shouldRetry(e)) {
+      appendRequestFailed(localAgent);
+    }
+  }
+
+  private boolean shouldRetry(Throwable e) {
+    return RaftUtil.isNetworkError(e);
+  }
+
+  protected void appendRequestFailed(ConsensusService localAgent) {
+    metrics.incAppendEntriesFailures();
+
+    // If we were the first one to fail and continue to retry
+    if (clearConsensusServiceAgent(localAgent)) {
+      // Sending no response to process, will automatically let it to recover from
+      // the last acked Edit id.
+      stateMachine.offer(new PeerAppendResponseEvent(null));
+    }
+  }
+
+  @Override
+  public void sendVoteRequestWithCallBack(final VoteRequest request) {
+    if (isStopped) {
+      if (LOG.isTraceEnabled()) {
+        LOG.trace(getSourceString() + " not sending " + request + " to "
+          + getTargetString() + "; because we are stopped!");
+      }
+      return;
+    }
+
+    final ConsensusService localAgent = getConsensusServiceAgent();
+    if (localAgent == null) {
+      voteRequestFailed(localAgent);
+      LOG.warn("Failed to get localAgent for " + this);
+      this.triggerRPCErrorEvent();
+      return;
+    }
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(getSourceString() + ": sending " + request + " to " +
+        getTargetString());
+    }
+
+    RaftUtil.getThriftClientManager().getNiftyChannel(localAgent).executeInIoThread(
+      new Runnable() {
+        @Override public void run() {
+          try {
+            ListenableFuture<VoteResponse> futureResponse =
+              localAgent.requestVote(request);
+            if (futureResponse == null) {
+              LOG.error(getSourceString() + ": got a null response when sending " +
+                request + " to " + getTargetString());
+              return;
+            }
+
+            Futures.addCallback(futureResponse, new FutureCallback<VoteResponse>() {
+              @Override
+              public void onSuccess(VoteResponse response) {
+                // Add the event to the state machine
+                getRaftContext().offerEvent(new VoteResponseEvent(response));
+              }
+
+              @Override
+              public void onFailure(Throwable e) {
+                LOG.error(request + " (" + getSourceString() + "-->" +
+                  getTargetString() + ") FAILED due to ", e);
+                voteRequestFailed(localAgent);
+              }
+            });
+          } catch (Throwable e) {
+            voteRequestFailed(localAgent);
+          }
+
+        }
+      });
+ }
+
+  protected void voteRequestFailed(ConsensusService localAgent) {
+    metrics.incVoteRequestFailures();
+    getRaftContext().offerEvent(new VoteResponseEvent(
+        new VoteResponse(this.getPeerServerName(),
+          HConstants.UNDEFINED_TERM_INDEX, VoteResponse.VoteResult.FAILURE)
+    ));
+    if (localAgent != null) {
+      clearConsensusServiceAgent(localAgent);
+    }
+  }
+
+  protected void triggerRPCErrorEvent() {
+    metrics.incRPCErrorEvents();
+    // Create an RPCError Event and add it to the SM
+    stateMachine.offer(new Event(PeerServerEventType.PEER_RPC_ERROR));
+  }
+
+  @Override
+  public PeerMetrics getMetrics() {
+    return metrics;
+  }
+
+  public void calculateAndSetAppendLag() {
+    getMetrics().setAppendEntriesLag(
+            latestRequest.getLogId(0).getIndex() - getLastEditID().getIndex());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerConsensusServer.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerConsensusServer.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerConsensusServer.java
new file mode 100644
index 0000000..eb8003f
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerConsensusServer.java
@@ -0,0 +1,152 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import com.facebook.nifty.client.FramedClientConnector;
+import com.facebook.nifty.duplex.TDuplexProtocolFactory;
+import com.facebook.swift.service.ThriftClient;
+import com.facebook.swift.service.ThriftClientConfig;
+import com.google.common.util.concurrent.ListenableFuture;
+import io.airlift.units.Duration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.consensus.quorum.MutableRaftContext;
+import org.apache.hadoop.hbase.consensus.server.ConsensusService;
+import org.apache.hadoop.hbase.consensus.util.RaftUtil;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+@NotThreadSafe
+public class PeerConsensusServer extends AbstractPeer {
+  private static Logger LOG = LoggerFactory.getLogger(PeerConsensusServer.class);
+
+  private ThriftClientConfig thriftClientConf;
+  private ThriftClient<ConsensusService> thriftClient;
+  private ListenableFuture<ConsensusService> futureConnection;
+  private ConsensusService agent;
+
+  public PeerConsensusServer(final HServerAddress address,
+                             final int rank,
+                             final MutableRaftContext replica,
+                             final Configuration conf) {
+    super(address, rank, replica, conf);
+    int connectionTimeout = conf.getInt(
+            HConstants.RAFT_PEERSERVER_CONNECTION_TIMEOUT_MS, 1000);
+    int readTimeout = conf.getInt(
+            HConstants.RAFT_PEERSERVER_READ_TIMEOUT_MS, 10000);
+    int writeTimeout = conf.getInt(
+            HConstants.RAFT_PEERSERVER_WRITE_TIMEOUT_MS, 10000);
+
+    thriftClientConf = new ThriftClientConfig()
+            .setConnectTimeout(new Duration(connectionTimeout,
+                    TimeUnit.MILLISECONDS))
+            .setReadTimeout(new Duration(readTimeout, TimeUnit.MILLISECONDS))
+            .setWriteTimeout(new Duration(writeTimeout, TimeUnit.MILLISECONDS));
+  }
+
+  @Override
+  public void stop() {
+    super.stop();
+    synchronized (this) {
+      if (futureConnection != null) {
+        futureConnection.cancel(true);
+      }
+      if (agent != null) {
+        try {
+          agent.close();
+        } catch (Exception e) {
+          LOG.error("Cannot close the agent. Error: ", e);
+        }
+      }
+    }
+  }
+
+  @Override
+  protected void resetConnection()
+    throws ExecutionException, InterruptedException, TimeoutException {
+    LOG.debug("Resetting the peer connection to " + this + "(" + getAddress() + ")");
+    thriftClient =
+            new ThriftClient<>(RaftUtil.getThriftClientManager(), ConsensusService.class,
+                    thriftClientConf, this.toString());
+    ListenableFuture<ConsensusService> futureConnection = thriftClient.open(
+      new FramedClientConnector(getAddress().getInetSocketAddress(),
+        TDuplexProtocolFactory.fromSingleFactory(
+          new TCompactProtocol.Factory())));
+    setAgent(futureConnection, futureConnection.get(
+      1000,
+      TimeUnit.MILLISECONDS));
+  }
+
+  @Override
+  protected boolean clearConsensusServiceAgent(ConsensusService localAgent) {
+    // TODO: double check with Nifty community to better handler exceptions
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(getRaftContext().getMyAddress() + " attempting to close "
+        + localAgent + "(" + getAddress() + ")");
+    }
+    try {
+      localAgent.close();
+    } catch (Exception e) {
+      LOG.error("Unable to close the connection: " + e);
+    } finally {
+      return resetAgentIfEqual(localAgent);
+    }
+  }
+
+  private synchronized ConsensusService getAgent() {
+    return this.agent;
+  }
+
+  private synchronized void setAgent(ListenableFuture<ConsensusService> futureConnection, ConsensusService agent) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(getRaftContext().getMyAddress() + " setting agent to " +
+        agent + "(" + getAddress() + ")" +  " using " + futureConnection);
+    }
+    this.futureConnection = futureConnection;
+    this.agent = agent;
+  }
+
+  private synchronized boolean resetAgentIfEqual(ConsensusService localAgent) {
+    boolean success = false;
+    if (agent == localAgent) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(getRaftContext().getMyAddress() + " resetting agent "
+          + agent + " to null" + "(" + getAddress() + ")");
+      }
+      agent = null;
+      success = true;
+    }
+    return success;
+  }
+
+  @Override
+  protected ConsensusService getConsensusServiceAgent() {
+    if (getAgent() == null) {
+      synchronized (this) {
+        if (getAgent() == null) {
+          for (int i = 0; i < connectionRetry; i++) {
+            try {
+              resetConnection();
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("New connection established from " + getRaftContext().getMyAddress()
+                  + " to " + this + ": agent = " + agent + "(" + getAddress() + ")");
+              }
+            } catch (Throwable t) {
+              LOG.error("Failed to reset the connection to " + this +  " to " +
+                getAddress() + " due to " + t);
+              try {
+                Thread.sleep(connectionRetryInterval * i);
+              } catch (InterruptedException e) {}
+            }
+          }
+        }
+      }
+    }
+    return getAgent();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServer.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServer.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServer.java
new file mode 100644
index 0000000..81dbd57
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServer.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+import org.apache.hadoop.hbase.consensus.rpc.VoteRequest;
+import org.apache.hadoop.hbase.regionserver.RaftEventListener;
+
+public interface PeerServer extends PeerServerMutableContext {
+  public void sendAppendEntries(AppendRequest request);
+  public void sendRequestVote(VoteRequest request);
+  public int getRank();
+  public void setRank(int rank);
+  public String getPeerServerName();
+  public Configuration getConf();
+  public void initialize();
+  public void stop();
+  public void registerDataStoreEventListener(RaftEventListener listener);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerImmutableContext.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerImmutableContext.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerImmutableContext.java
new file mode 100644
index 0000000..1f90a0c
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerImmutableContext.java
@@ -0,0 +1,10 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import org.apache.hadoop.hbase.consensus.protocol.EditId;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+
+public interface PeerServerImmutableContext {
+  EditId getLastEditID();
+  AppendRequest getLatestRequest();
+  int getRank();
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerMutableContext.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerMutableContext.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerMutableContext.java
new file mode 100644
index 0000000..e98a1e2
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerServerMutableContext.java
@@ -0,0 +1,23 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.metrics.PeerMetrics;
+import org.apache.hadoop.hbase.consensus.protocol.EditId;
+import org.apache.hadoop.hbase.consensus.quorum.MutableRaftContext;
+import org.apache.hadoop.hbase.consensus.quorum.Timer;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+import org.apache.hadoop.hbase.consensus.rpc.VoteRequest;
+
+public interface PeerServerMutableContext extends PeerServerImmutableContext {
+  void sendVoteRequestWithCallBack(VoteRequest request);
+  void sendAppendRequestWithCallBack(AppendRequest request);
+  MutableRaftContext getQuorumContext();
+  void enqueueEvent(final Event e);
+  void setLastEditID(final EditId id);
+  void stop();
+  void resetPeerContext();
+  void updatePeerAvailabilityStatus(boolean isAvailable);
+  PeerMetrics getMetrics();
+  void setLatestRequest(AppendRequest latestRequest);
+  void calculateAndSetAppendLag();
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerStateMachine.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerStateMachine.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerStateMachine.java
new file mode 100644
index 0000000..90d3493
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/PeerStateMachine.java
@@ -0,0 +1,84 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import org.apache.hadoop.hbase.consensus.fsm.*;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerServerEventType;
+import org.apache.hadoop.hbase.consensus.server.peer.states.*;
+import org.apache.hadoop.hbase.consensus.server.peer.transition.PeerServerTransitionType;
+
+public class PeerStateMachine extends FiniteStateMachine {
+
+  public PeerStateMachine(final String name, final PeerServer peerServer) {
+    super(name);
+
+    // Define all the states:
+    State start = new Start(peerServer);
+    State peerFollower = new PeerFollower(peerServer);
+    State handleVoteResponse = new PeerHandleVoteResponse(peerServer);
+    State sendVoteRequest = new PeerSendVoteRequest(peerServer);
+    State handleAppendResponse = new PeerHandleAppendResponse(peerServer);
+    State sendAppendRequest = new PeerSendAppendRequest(peerServer);
+    State handleRPCError = new PeerHandleRPCError(peerServer);
+    State halt = new PeerServerState(PeerServerStateType.HALT, peerServer);
+
+    // Define all the transitions:
+    Transition onStart =
+            new Transition(PeerServerTransitionType.ON_START,
+                    new OnEvent(PeerServerEventType.START));
+
+    Transition onAppendRequestReceived =
+            new Transition(PeerServerTransitionType.ON_APPEND_REQUEST,
+                    new OnEvent(PeerServerEventType.PEER_APPEND_REQUEST_RECEIVED));
+
+    Transition onAppendResponseReceived =
+            new Transition(PeerServerTransitionType.ON_APPEND_RESPONSE,
+                    new OnEvent(PeerServerEventType.PEER_APPEND_RESPONSE_RECEIVED));
+
+    Transition onVoteRequestReceived =
+            new Transition(PeerServerTransitionType.ON_VOTE_REQUEST,
+                    new OnEvent(PeerServerEventType.PEER_VOTE_REQUEST_RECEIVED));
+
+    Transition onVoteResponseReceived =
+            new Transition(PeerServerTransitionType.ON_VOTE_RESPONSE,
+                    new OnEvent(PeerServerEventType.PEER_VOTE_RESPONSE_RECEIVED));
+
+    Transition onRPCErrorReceived =
+            new Transition(PeerServerTransitionType.ON_RPC_ERROR,
+                    new OnEvent(PeerServerEventType.PEER_RPC_ERROR));
+
+    Transition peerReachable =
+            new Transition(PeerServerTransitionType.ON_PEER_IS_REACHABLE,
+                    new OnEvent(PeerServerEventType.PEER_REACHABLE));
+
+    Transition unConditional =
+            new Transition(PeerServerTransitionType.UNCONDITIONAL,
+                    new Unconditional());
+
+    Transition onHalt = new Transition(PeerServerTransitionType.ON_HALT,
+                    new OnEvent(PeerServerEventType.HALT));
+
+    // Add the transitions and states into the state machine
+    addTransition(start, peerFollower, onStart);
+
+    addTransition(peerFollower, handleVoteResponse, onVoteResponseReceived);
+    addTransition(peerFollower, handleAppendResponse, onAppendResponseReceived);
+    addTransition(peerFollower, handleAppendResponse, peerReachable);
+    addTransition(peerFollower, sendVoteRequest, onVoteRequestReceived);
+    addTransition(peerFollower, sendAppendRequest, onAppendRequestReceived);
+    addTransition(peerFollower, handleRPCError, onRPCErrorReceived);
+    addTransition(peerFollower, halt, onHalt);
+
+    addTransition(handleAppendResponse, peerFollower, unConditional);
+
+    addTransition(sendVoteRequest, peerFollower, unConditional);
+
+    addTransition(sendAppendRequest, peerFollower, unConditional);
+
+    addTransition(handleVoteResponse, peerFollower, unConditional);
+
+    addTransition(handleRPCError, peerFollower, peerReachable);
+    addTransition(handleRPCError, sendVoteRequest, onVoteRequestReceived);
+
+    // Set the initial state
+    setStartState(start);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/ReconnectTimeoutCallback.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/ReconnectTimeoutCallback.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/ReconnectTimeoutCallback.java
new file mode 100644
index 0000000..e6809a4
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/ReconnectTimeoutCallback.java
@@ -0,0 +1,21 @@
+package org.apache.hadoop.hbase.consensus.server.peer;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.quorum.TimeoutEventHandler;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerServerEventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReconnectTimeoutCallback implements TimeoutEventHandler {
+  private static final Logger
+    LOG = LoggerFactory.getLogger(ReconnectTimeoutCallback.class);
+  private final PeerServer peer;
+
+  public ReconnectTimeoutCallback(final PeerServer peer) {
+    this.peer = peer;
+  }
+  public void onTimeout() {
+    LOG.debug(peer + " Reconnect timeout Triggered");
+    peer.enqueueEvent(new Event(PeerServerEventType.PEER_REACHABLE));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendRequestEvent.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendRequestEvent.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendRequestEvent.java
new file mode 100644
index 0000000..0417feb
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendRequestEvent.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer.events;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+
+public class PeerAppendRequestEvent extends Event {
+
+  private final AppendRequest request;
+
+  public PeerAppendRequestEvent(AppendRequest request) {
+    super(PeerServerEventType.PEER_APPEND_REQUEST_RECEIVED);
+    this.request = request;
+  }
+
+  public AppendRequest getAppendRequest() {
+    return this.request;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendResponseEvent.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendResponseEvent.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendResponseEvent.java
new file mode 100644
index 0000000..410cd97
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerAppendResponseEvent.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer.events;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.rpc.AppendResponse;
+
+public class PeerAppendResponseEvent extends Event {
+
+  private final AppendResponse response;
+
+  public PeerAppendResponseEvent(AppendResponse response) {
+    super(PeerServerEventType.PEER_APPEND_RESPONSE_RECEIVED);
+    this.response = response;
+  }
+
+  public AppendResponse getAppendResponse() {
+    return this.response;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerServerEventType.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerServerEventType.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerServerEventType.java
new file mode 100644
index 0000000..23f68c4
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerServerEventType.java
@@ -0,0 +1,16 @@
+package org.apache.hadoop.hbase.consensus.server.peer.events;
+
+import org.apache.hadoop.hbase.consensus.fsm.EventType;
+
+public enum PeerServerEventType implements EventType {
+  NONE,
+  START,
+  PEER_VOTE_REQUEST_RECEIVED,
+  PEER_VOTE_RESPONSE_RECEIVED,
+  PEER_APPEND_REQUEST_RECEIVED,
+  PEER_APPEND_RESPONSE_RECEIVED,
+  PEER_RPC_ERROR,
+  PEER_REACHABLE,
+  HALT,
+  MAX
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteRequestEvent.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteRequestEvent.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteRequestEvent.java
new file mode 100644
index 0000000..43a90a0
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteRequestEvent.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.hbase.consensus.server.peer.events;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.rpc.VoteRequest;
+
+public class PeerVoteRequestEvent extends Event {
+
+  private final VoteRequest request;
+
+  public PeerVoteRequestEvent(VoteRequest request) {
+    super(PeerServerEventType.PEER_VOTE_REQUEST_RECEIVED);
+    this.request = request;
+  }
+
+  public VoteRequest getVoteRequest() {
+    return this.request;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteResponseEvent.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteResponseEvent.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteResponseEvent.java
new file mode 100644
index 0000000..f5cf85c
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/events/PeerVoteResponseEvent.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer.events;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.rpc.VoteResponse;
+
+public class PeerVoteResponseEvent extends Event {
+
+  private final VoteResponse response;
+
+  public PeerVoteResponseEvent(VoteResponse response) {
+    super(PeerServerEventType.PEER_VOTE_RESPONSE_RECEIVED);
+    this.response = response;
+  }
+
+  public VoteResponse getVoteResponse() {
+    return this.response;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerFollower.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerFollower.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerFollower.java
new file mode 100644
index 0000000..d32c471
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerFollower.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+public class PeerFollower extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerFollower.class);
+
+  public PeerFollower(PeerServerMutableContext context) {
+    super(PeerServerStateType.PEER_FOLLOWER, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    // TODO
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleAppendResponse.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleAppendResponse.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleAppendResponse.java
new file mode 100644
index 0000000..40205a5
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleAppendResponse.java
@@ -0,0 +1,282 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.fsm.FSMLargeOpsExecutorService;
+import org.apache.hadoop.hbase.consensus.log.TransactionLogManager;
+import org.apache.hadoop.hbase.consensus.protocol.ConsensusHost;
+import org.apache.hadoop.hbase.consensus.protocol.EditId;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+import org.apache.hadoop.hbase.consensus.rpc.AppendResponse;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendRequestEvent;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendResponseEvent;
+import org.apache.hadoop.hbase.util.Arena;
+import org.apache.hadoop.hbase.util.MemoryBuffer;
+import org.apache.hadoop.hbase.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class PeerHandleAppendResponse extends PeerServerAsyncState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerHandleAppendResponse.class);
+  private static int globalMaxBatchLogs = HConstants.RAFT_BATCH_APPEND_MAX_EDITS_DEFAULT;
+  private static long globalMaxBatchBytes = HConstants.RAFT_BATCH_APPEND_MAX_BYTES_DEFAULT;
+  private static boolean batchSameTerm = false;
+  private final List<MemoryBuffer> buffers = new ArrayList<>();
+  private final Arena arena;
+  private ListenableFuture<?> createAppendRequestFromLogFuture = null;
+  private static long maxwaitToReadFromDiskNs;
+
+  public PeerHandleAppendResponse(PeerServerMutableContext context) {
+    super(PeerServerStateType.HANDLE_APPEND_RESPONSE, context);
+    arena = context.getQuorumContext().getArena();
+    globalMaxBatchLogs = getConf().getInt(
+      HConstants.CONSENSUS_PUSH_APPEND_MAX_BATCH_LOGS_KEY,
+      globalMaxBatchLogs);
+    globalMaxBatchBytes = getConf().getLong(
+      HConstants.CONSENSUS_PUSH_APPEND_MAX_BATCH_BYTES_KEY,
+      globalMaxBatchBytes);
+    batchSameTerm = getConf().getBoolean(
+      HConstants.CONSENSUS_PUSH_APPEND_BATCH_SAME_TERM_KEY,
+      false);
+    maxwaitToReadFromDiskNs =
+        getConf().getLong(HConstants.MAXWAIT_TO_READ_FROM_DISK_NS,
+          HConstants.MAXWAIT_TO_READ_FROM_DISK_NS_DEFAULT);
+  }
+
+  public static void setGlobalMaxBatchLogs(int n) {
+    globalMaxBatchLogs = Math.max(1, n);
+  }
+
+  private Configuration getConf() {
+    return c.getQuorumContext().getConf();
+  }
+
+  /**
+   *  batchSameTerm
+   *
+   *  Whether to batch edits that have the same term.
+   */
+  public boolean batchSameTerm() {
+    return batchSameTerm;
+  }
+
+  public int getMaxBatchLogs() {
+    return globalMaxBatchLogs;
+  }
+
+  public long getMaxBatchBytes() {
+    return globalMaxBatchBytes;
+  }
+
+  @Override
+  public boolean isComplete() {
+    return (createAppendRequestFromLogFuture == null ||
+      createAppendRequestFromLogFuture.isDone());
+  }
+
+  @Override
+  public ListenableFuture<?> getAsyncCompletion() {
+    return createAppendRequestFromLogFuture;
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    createAppendRequestFromLogFuture = null;
+
+    // Nothing to do if you are not the leader
+    if (!c.getQuorumContext().isLeader()) {
+      return;
+    }
+
+    // Add the event to the raft state machine
+    // update the last successfully committed edit by the peer
+    if (e instanceof PeerAppendResponseEvent) {
+      final AppendResponse response = ((PeerAppendResponseEvent)e).getAppendResponse();
+      if (response != null) {
+        c.setLastEditID(response.getPrevEditID());
+        // Update the peer's availability
+        c.updatePeerAvailabilityStatus(true);
+      } else {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(c + " got null as AppendResponse");
+        }
+      }
+    }
+
+    final EditId startEditId = c.getLastEditID();
+    final EditId logManagerLastEditID =
+      c.getQuorumContext().getLogManager().getLastEditID();
+
+    // If the log manager has new entries
+    if (logManagerLastEditID.compareTo(startEditId) > 0) {
+      Runnable createAppendRequestRunnable = new Runnable() {
+        @Override
+        public void run() {
+          AppendRequest nextRequest = createAppendRequestFromLog(startEditId,
+            logManagerLastEditID);
+
+          if (nextRequest != null) {
+            if (LOG.isTraceEnabled()) {
+              LOG.trace(c + " is sending " + nextRequest);
+            }
+            c.enqueueEvent(new PeerAppendRequestEvent(nextRequest));
+          }
+        }
+      };
+
+      createAppendRequestFromLogFuture =
+        FSMLargeOpsExecutorService.submitToReadOpsThreadPool(
+          createAppendRequestRunnable);
+    } else if (c.getLatestRequest() != null &&
+      c.getLatestRequest().getLogId(0).compareTo(startEditId) > 0) {
+      // We have seen new entries which might have not been made in the log manager.
+      // Lets send it.
+      AppendRequest nextRequest = c.getLatestRequest();
+      if (nextRequest != null) {
+        if (LOG.isTraceEnabled()) {
+          LOG.trace(c + " is sending " + nextRequest);
+        }
+        c.enqueueEvent(new PeerAppendRequestEvent(nextRequest));
+      }
+    }
+  }
+
+  /**
+   * Depending upon the batching parameters, it fetches the next set of transactions
+   * for the log manager.
+   *
+   * @param startEditId EditId after which, transactions will be fetched
+   * @param logManagerLastEditID latest EditId reported by the log manager
+   * @return AppendRequest, or null in case of error
+   */
+  private AppendRequest createAppendRequestFromLog(final EditId startEditId,
+                                                   final EditId logManagerLastEditID) {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(c + " adding lagging entries starting from " + startEditId +
+        "; log manager has " + logManagerLastEditID);
+    }
+
+    if (arena != null) {
+      // Reset the buffers as we are going to read new transactions
+      for (MemoryBuffer buffer : buffers) {
+        arena.freeByteBuffer(buffer);
+      }
+    }
+
+    buffers.clear();
+
+    long start = System.nanoTime();
+
+    // If the peer is at a lesser or equal term as yours and if there are more
+    // entries present in the log, then send the missing transactions.
+    List<ByteBuffer> edits = new ArrayList<>();
+    List<EditId> logIds = new ArrayList<EditId>();
+    long nbytes = 0;
+
+    EditId lastEditID = startEditId;
+    int maxEdits = getMaxBatchLogs();
+    long maxBytes = getMaxBatchBytes();
+
+    ConsensusHost leader = c.getQuorumContext().getLeader();
+    // we could check if leader corresponds to c.getQuorumContext().
+    // But it's unnecessary.
+    collect_edits_loop:
+    while (c.getQuorumContext().isLeader() &&
+      logManagerLastEditID.compareTo(lastEditID) > 0 &&
+      nbytes < maxBytes && edits.size() < maxEdits &&
+      // time taken so far is smaller than defined threshold
+        (System.nanoTime() - start) < maxwaitToReadFromDiskNs) {
+      try {
+        Pair<EditId, MemoryBuffer> idTxn =
+          c.getQuorumContext().getLogManager().getNextEditIdTransaction(
+          c.toString(), lastEditID.getIndex(), arena);
+
+        if (idTxn == null ||
+          idTxn.getFirst() == null ||
+          idTxn.getFirst().equals(TransactionLogManager.UNDEFINED_EDIT_ID)) {
+          LOG.error(c.getQuorumContext() + " could not locate next id after "
+            + lastEditID + ". Returning.." + idTxn);
+          break collect_edits_loop;
+        }
+
+        buffers.add(idTxn.getSecond());
+        // We successfully found the entry, lets update the sharedPoolBuffer
+        EditId nextEditId = idTxn.getFirst();
+
+        // we currently cannot handle logs with differing terms on the receiver side
+        if (batchSameTerm() && logIds.size() > 0 &&
+          nextEditId.getTerm() != logIds.get(0).getTerm()) {
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("---- " + c + " stop at lagging entry " + nextEditId +
+              "[prev=" + lastEditID + "] due to changing terms from "
+              + logIds.get(0));
+          }
+          break collect_edits_loop;
+        } else {
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("---- " + c + " trying to add lagging entry: " +
+              nextEditId + "[prev=" + lastEditID + "]");
+          }
+        }
+
+        ByteBuffer edit = idTxn.getSecond().getBuffer();
+        edits.add(edit);
+        logIds.add(nextEditId);
+        nbytes += edit.remaining();
+        lastEditID = nextEditId;
+      } catch (IOException ex) {
+        LOG.error(
+          c.getQuorumContext() + " is unable to read next transaction " +
+            "for previous id " + lastEditID + " for " + c, ex);
+        break collect_edits_loop;
+      }
+    }
+
+    if (edits.size() == 0) {
+      LOG.warn("NOTHING to APPEND!");
+      return null;
+    }
+
+    if (!leader.equals(c.getQuorumContext().getLeader())) {
+      // Note that this covers the following cases
+      // 1. I'm a leader both before and after the loop; but there was at least
+      // one re-election in between.
+      // 2. I was a leader before the loop and no longer the leader afterwards.
+      LOG.warn(c.getQuorumContext() + " aborts sending AppendRequest because " +
+        "the leader has changed from " + leader + " to " +
+        c.getQuorumContext().getLeader());
+      return null;
+    }
+
+    long batchCommitIndex = Math.max(logIds.get(logIds.size() - 1).getIndex(),
+      c.getQuorumContext().getCommittedEdit().getIndex());
+
+    // Update the metrics
+    long elapsed = System.nanoTime() - start;
+    c.getMetrics().getBatchRecoverySize().add(logIds.size());
+    c.getMetrics().getBatchRecoveryLatency().add(elapsed, TimeUnit.NANOSECONDS);
+
+    // Create request from nextEditId;
+    return (new AppendRequest(
+      c.getQuorumContext().getQuorumName(),
+      leader,
+      false,
+      batchCommitIndex,
+      c.getQuorumContext().getMinUnPersistedIndexAcrossQuorum(),
+      startEditId,
+      logIds,
+      edits
+    ));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleRPCError.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleRPCError.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleRPCError.java
new file mode 100644
index 0000000..33f0575
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleRPCError.java
@@ -0,0 +1,27 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendResponseEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PeerHandleRPCError extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerHandleRPCError.class);
+
+  public PeerHandleRPCError(PeerServerMutableContext context) {
+    super(PeerServerStateType.HANDLE_RPC_ERROR, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    c.updatePeerAvailabilityStatus(false);
+  }
+
+  @Override
+  public void onExit(final Event e) {
+    super.onExit(e);
+    c.enqueueEvent(new PeerAppendResponseEvent(null));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleVoteResponse.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleVoteResponse.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleVoteResponse.java
new file mode 100644
index 0000000..c977829
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerHandleVoteResponse.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+public class PeerHandleVoteResponse extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerHandleVoteResponse.class);
+
+  public PeerHandleVoteResponse(PeerServerMutableContext context) {
+    super(PeerServerStateType.HANDLE_VOTE_RESPONSE, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    // TODO
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerRecovery.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerRecovery.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerRecovery.java
new file mode 100644
index 0000000..14dbabd
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerRecovery.java
@@ -0,0 +1,19 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+public class PeerRecovery extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerRecovery.class);
+
+  public PeerRecovery(PeerServerMutableContext context) {
+    super(PeerServerStateType.RECOVERY, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    // TODO
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendAppendRequest.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendAppendRequest.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendAppendRequest.java
new file mode 100644
index 0000000..7b9e095
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendAppendRequest.java
@@ -0,0 +1,50 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.rpc.AppendRequest;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerAppendRequestEvent;
+
+public class PeerSendAppendRequest extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerSendAppendRequest.class);
+
+  public PeerSendAppendRequest(PeerServerMutableContext context) {
+    super(PeerServerStateType.SEND_APPEND_REQUEST, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+
+    final AppendRequest request = (((PeerAppendRequestEvent)e)).getAppendRequest();
+
+    // Seen a new request, update the latest request
+    AppendRequest lastSeenAppendRequest = c.getLatestRequest();
+    if (lastSeenAppendRequest == null ||
+      lastSeenAppendRequest.getLogId(0).compareTo(request.getLogId(0)) < 0) {
+      c.setLatestRequest(request);
+    }
+    if (request.isTraceable()) {
+      LOG.debug("[AppendRequest Trace] " + c.toString() + " is processing " + request + " " +
+        "lastRequest " + c.getLatestRequest() + " lastEditID " + c.getLastEditID());
+    }
+    c.calculateAndSetAppendLag();
+
+    // Send the request only if a) the latest EditId ack'ed by the peer is the
+    // as the EditId of the previous request or b) if the latest ack'ed Id is
+    // the same as the current Id.
+    //
+    // Case b is important if no majority has been reached yet on the request,
+    // and it needs to be retried. It needs to be resent to the peer to act as a
+    // heartbeat and keep its progress timer from firing. Failing to do so would
+    // lead to unnecessary leader election.
+    if (request.getPrevLogId().compareTo(c.getLastEditID()) == 0 ||
+            request.getLogId(0).compareTo(c.getLastEditID()) == 0) {
+      c.sendAppendRequestWithCallBack(request);
+    } else if (request.isTraceable()) {
+        LOG.debug("[AppendRequest Trace] Not able to send the new append request out: " +
+          "request: " + request + " and the lastEditID is " + c.getLastEditID());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendVoteRequest.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendVoteRequest.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendVoteRequest.java
new file mode 100644
index 0000000..28f6b35
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerSendVoteRequest.java
@@ -0,0 +1,22 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+import org.apache.hadoop.hbase.consensus.server.peer.events.PeerVoteRequestEvent;
+
+public class PeerSendVoteRequest extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(PeerSendVoteRequest.class);
+
+  public PeerSendVoteRequest(PeerServerMutableContext context) {
+    super(PeerServerStateType.SEND_VOTE_REQUEST, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+    PeerVoteRequestEvent voteRequestEvent = (PeerVoteRequestEvent)e;
+    this.c.sendVoteRequestWithCallBack(voteRequestEvent.getVoteRequest());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerAsyncState.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerAsyncState.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerAsyncState.java
new file mode 100644
index 0000000..6b13fa9
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerAsyncState.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+/**
+ * A {@link PeerServerState} which has async tasks.
+ */
+public class PeerServerAsyncState extends PeerServerState {
+  public PeerServerAsyncState(final PeerServerStateType t,
+                              PeerServerMutableContext c) {
+    super(t, c);
+  }
+
+  @Override
+  public boolean isAsyncState() {
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerState.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerState.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerState.java
new file mode 100644
index 0000000..a21201b
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerState.java
@@ -0,0 +1,23 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.fsm.State;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+public class PeerServerState extends State {
+  private static Logger LOG = LoggerFactory.getLogger(PeerServerState.class);
+
+  protected PeerServerMutableContext c;
+  public PeerServerState(final PeerServerStateType t, PeerServerMutableContext c) {
+    super(t);
+    this.c = c;
+  }
+
+  @Override
+  public void onEntry(final Event e) {}
+
+  @Override
+  public void onExit(final Event e) {}
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerStateType.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerStateType.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerStateType.java
new file mode 100644
index 0000000..6f3a781
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/PeerServerStateType.java
@@ -0,0 +1,16 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+
+import org.apache.hadoop.hbase.consensus.fsm.StateType;
+
+public enum PeerServerStateType implements StateType {
+  START,
+  PEER_FOLLOWER,
+  SEND_VOTE_REQUEST,
+  HANDLE_VOTE_RESPONSE,
+  SEND_APPEND_REQUEST,
+  HANDLE_APPEND_RESPONSE,
+  HANDLE_RPC_ERROR,
+  RECOVERY,
+  HALT,
+  MAX
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/Start.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/Start.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/Start.java
new file mode 100644
index 0000000..0d76636
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/states/Start.java
@@ -0,0 +1,18 @@
+package org.apache.hadoop.hbase.consensus.server.peer.states;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.consensus.fsm.Event;
+import org.apache.hadoop.hbase.consensus.server.peer.PeerServerMutableContext;
+
+public class Start extends PeerServerState {
+  private static Logger LOG = LoggerFactory.getLogger(Start.class);
+
+  public Start(PeerServerMutableContext context) {
+    super(PeerServerStateType.START, context);
+  }
+
+  @Override
+  public void onEntry(final Event e) {
+    super.onEntry(e);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/transition/PeerServerTransitionType.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/transition/PeerServerTransitionType.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/transition/PeerServerTransitionType.java
new file mode 100644
index 0000000..1958bed
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/server/peer/transition/PeerServerTransitionType.java
@@ -0,0 +1,22 @@
+package org.apache.hadoop.hbase.consensus.server.peer.transition;
+
+import org.apache.hadoop.hbase.consensus.fsm.TransitionType;
+
+public enum PeerServerTransitionType implements TransitionType {
+  NONE,
+  UNCONDITIONAL,
+  ON_START,
+  ON_VOTE_REQUEST,
+  ON_VOTE_RESPONSE,
+
+  ON_APPEND_REQUEST,
+  ON_APPEND_RESPONSE,
+
+  ON_APPEND_NACK_RESPONSE,
+  ON_APPEND_ACK_RESPONSE,
+  ON_PEER_IS_REACHABLE,
+  ON_RPC_ERROR,
+
+  ON_HALT,
+  MAX
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eca32aa4/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/util/RaftUtil.java
----------------------------------------------------------------------
diff --git a/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/util/RaftUtil.java b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/util/RaftUtil.java
new file mode 100644
index 0000000..d73cb8e
--- /dev/null
+++ b/hbase-consensus/src/main/java/org/apache/hadoop/hbase/consensus/util/RaftUtil.java
@@ -0,0 +1,107 @@
+package org.apache.hadoop.hbase.consensus.util;
+
+import com.facebook.nifty.client.NettyClientConfig;
+import com.facebook.nifty.client.NettyClientConfigBuilder;
+import com.facebook.nifty.client.NiftyClient;
+import com.facebook.swift.codec.ThriftCodecManager;
+import com.facebook.swift.service.ThriftClientEventHandler;
+import com.facebook.swift.service.ThriftClientManager;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.consensus.quorum.AggregateTimer;
+import org.apache.hadoop.hbase.consensus.quorum.RepeatingTimer;
+import org.apache.hadoop.hbase.consensus.quorum.TimeoutEventHandler;
+import org.apache.hadoop.hbase.consensus.quorum.Timer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.jboss.netty.channel.socket.nio.NioSocketChannelConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class RaftUtil {
+  private static final Logger LOG = LoggerFactory.getLogger(RaftUtil.class);
+  private static ThriftClientManager clientManager;
+
+  static {
+    try {
+      NettyClientConfigBuilder clientConfigBuilder =
+        NettyClientConfig.newBuilder();
+      final NioSocketChannelConfig socketConfig =
+        clientConfigBuilder.getSocketChannelConfig();
+
+      socketConfig.setKeepAlive(true);
+      socketConfig.setTcpNoDelay(true);
+      clientConfigBuilder.setBossThreadCount(2);
+
+      clientManager = new ThriftClientManager(
+        new ThriftCodecManager(),
+        new NiftyClient(clientConfigBuilder.build()),
+        ImmutableSet.<ThriftClientEventHandler>of());
+    } catch (Throwable t) {
+      LOG.error("Unable to initialize ThriftClientManager.", t);
+      throw t;
+    }
+  }
+
+  public static HRegionInfo createDummyRegionInfo(String region) {
+    return createDummyRegionInfo(region, null);
+  }
+
+  public static HRegionInfo createDummyRegionInfo(String region, Map<HServerAddress,
+    Integer> peers) {
+    HRegionInfo regionInfo = new HRegionInfo(new HTableDescriptor(region),
+        Bytes.toBytes("00000000"), Bytes.toBytes("ffffffff"), false, 1000,
+        null, null);
+    Map<String, Map<HServerAddress, Integer>> peerMap = new HashMap<>();
+    peerMap.put(HRegionInfo.LOCAL_DC_KEY, peers);
+    regionInfo.setPeers(peerMap);
+    return regionInfo;
+  }
+
+  public static <T> String listToString(List<T> list) {
+    if (list == null) {
+      return null;
+    }
+    return Joiner.on(", ").useForNull("null").join(list);
+  }
+
+  public static HServerAddress getHRegionServerAddress(HServerAddress
+                                                            localConsensusServerAddress) {
+    return new HServerAddress(localConsensusServerAddress.getBindAddress(),
+      localConsensusServerAddress.getPort() - HConstants.CONSENSUS_SERVER_PORT_JUMP);
+  }
+
+  public static HServerAddress getLocalConsensusAddress(HServerAddress
+                                                         regionServerAddress) {
+    return new HServerAddress(regionServerAddress.getBindAddress(),
+      regionServerAddress.getPort() + HConstants.CONSENSUS_SERVER_PORT_JUMP);
+  }
+
+  public static ThriftClientManager getThriftClientManager() {
+    return clientManager;
+  }
+
+  public static Timer createTimer(boolean useAggregateTimer, final String name,
+                                  final long delay, TimeUnit unit,
+                                  final TimeoutEventHandler callback,
+                                  final AggregateTimer aggregateTimer) {
+    if (useAggregateTimer) {
+      return aggregateTimer.createTimer(name, delay, unit, callback);
+    }
+    return new RepeatingTimer(name, delay, unit, callback);
+  }
+
+  public static boolean isNetworkError(Throwable e) {
+    return 
+      e instanceof org.apache.thrift.transport.TTransportException
+      || e instanceof com.facebook.swift.service.RuntimeTTransportException;
+  }
+}