You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@omid.apache.org by fp...@apache.org on 2016/05/11 18:06:38 UTC

[31/50] [abbrv] incubator-omid git commit: Remove reset()

Remove reset()

Change-Id: I449fa5a71e41f6a57fe654c5f8cf2344de9be0f6


Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/43261a3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/43261a3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/43261a3c

Branch: refs/heads/master
Commit: 43261a3c628a96ccd1c9fb8118922ecd6817acb6
Parents: 3e5b674
Author: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Authored: Thu Apr 28 14:05:19 2016 -0700
Committer: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Committed: Tue May 3 16:19:17 2016 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/omid/tso/Batch.java    |  11 +-
 .../java/org/apache/omid/tso/BatchPool.java     |  35 ++--
 .../apache/omid/tso/PersistenceProcessor.java   |   2 -
 .../omid/tso/PersistenceProcessorHandler.java   |  40 ++--
 .../omid/tso/PersistenceProcessorImpl.java      |  14 +-
 .../org/apache/omid/tso/ReplyProcessor.java     |   4 +-
 .../org/apache/omid/tso/ReplyProcessorImpl.java |  29 +--
 .../apache/omid/tso/RequestProcessorImpl.java   |   2 +-
 .../java/org/apache/omid/tso/TestBatch.java     |  14 +-
 .../omid/tso/TestPersistenceProcessor.java      | 207 +++++++++++--------
 10 files changed, 177 insertions(+), 181 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/Batch.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/Batch.java b/tso-server/src/main/java/org/apache/omid/tso/Batch.java
index 8f665e1..e778eee 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/Batch.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/Batch.java
@@ -20,9 +20,13 @@ package org.apache.omid.tso;
 import com.google.common.base.Preconditions;
 import org.apache.omid.tso.PersistEvent.Type;
 import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class Batch {
 
+    private static final Logger LOG = LoggerFactory.getLogger(Batch.class);
+
     private final PersistEvent[] events;
     private final int maxBatchSize;
     private final BatchPool batchPool;
@@ -37,6 +41,7 @@ public class Batch {
 
     Batch(int size, int id, BatchPool batchPool) {
         Preconditions.checkArgument(size > 0, "Size must be positive");
+        LOG.info("Batch id {} created with size {}", id, size);
         this.maxBatchSize = size;
         this.batchPool = batchPool;
         this.id = id;
@@ -118,7 +123,7 @@ public class Batch {
 
     }
 
-    void sendReply(ReplyProcessor reply, RetryProcessor retryProc, long batchID, boolean isTSOInstanceMaster) {
+    void sendReply(ReplyProcessor reply, RetryProcessor retryProc, long batchID) {
         int i = 0;
         while (i < numEvents) {
             PersistEvent e = events[i];
@@ -130,7 +135,7 @@ public class Batch {
                 events[numEvents - 1] = tmp;
                 if (numEvents == 1) {
                     clear();
-                    reply.batchResponse(null, batchID, !isTSOInstanceMaster);
+                    reply.batchResponse(null, batchID);
                     return;
                 }
                 numEvents--;
@@ -139,7 +144,7 @@ public class Batch {
             i++;
         }
 
-        reply.batchResponse(this, batchID, !isTSOInstanceMaster);
+        reply.batchResponse(this, batchID);
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/BatchPool.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/BatchPool.java b/tso-server/src/main/java/org/apache/omid/tso/BatchPool.java
index 6b5307e..88cfd1e 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/BatchPool.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/BatchPool.java
@@ -17,23 +17,32 @@
  */
 package org.apache.omid.tso;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import javax.inject.Inject;
 import java.util.Stack;
 
 public class BatchPool {
 
-    final private Batch[] batches;
-    final private int poolSize;
-    final private Stack<Integer> availableBatches;
+    private static final Logger LOG = LoggerFactory.getLogger(BatchPool.class);
+
+    private final Batch[] batches;
+    private final int poolSize;
+    private final Stack<Integer> availableBatches;
 
     @Inject
     public BatchPool(TSOServerConfig config) {
 
         int numBuffersPerHandler = (config.getNumBuffersPerHandler() >= 2) ? config.getNumBuffersPerHandler() : 2;
         poolSize = config.getPersistHandlerNum() * numBuffersPerHandler;
+        LOG.info("Config param maxBatchSize {}", config.getMaxBatchSize());
+        LOG.info("Persistent Handlers {} ; Buffers per handler {}", config.getPersistHandlerNum(), numBuffersPerHandler);
         batches = new Batch[poolSize];
-        int batchSize = (config.getMaxBatchSize() / config.getPersistHandlerNum() > 0) ? (config.getMaxBatchSize() / config.getPersistHandlerNum()) : 2;
+        // TODO The + 1 is because the low watermark is added in each flush and needs one extra position. Fix and clarify initialization
+        int batchSize = (config.getMaxBatchSize() / config.getPersistHandlerNum() > 0) ? (config.getMaxBatchSize() / config.getPersistHandlerNum()) + 1 : 2;
 
+        LOG.info("Creating {} Batches with {} elements each", poolSize, batchSize);
         for (int i = 0; i < poolSize; i++) {
             batches[i] = new Batch(batchSize, i, this);
         }
@@ -48,7 +57,7 @@ public class BatchPool {
 
     Batch getNextEmptyBatch() throws InterruptedException {
 
-        synchronized (availableBatches) {
+        synchronized (availableBatches) { // TODO Synchronized can be put at the method level
             while (availableBatches.isEmpty()) {
                 availableBatches.wait();
             }
@@ -61,25 +70,11 @@ public class BatchPool {
 
     void notifyEmptyBatch(int batchIdx) {
 
-        synchronized (availableBatches) {
+        synchronized (availableBatches) { // TODO Synchronized can be put at the method level
             availableBatches.push(batchIdx);
             availableBatches.notify();
         }
 
     }
 
-    public void reset() {
-
-        for (int i = 0; i < poolSize; i++) {
-            batches[i].clear();
-        }
-
-        availableBatches.clear();
-
-        for (int i = (poolSize - 1); i >= 0; i--) {
-            availableBatches.push(i);
-        }
-
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
index 823198b..47dc56b 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
@@ -39,6 +39,4 @@ interface PersistenceProcessor {
     // TODO The name of this method is weird. Rename to "persist"
     void persistFlush() throws InterruptedException;
 
-    void reset() throws InterruptedException;
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorHandler.java b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorHandler.java
index 36b1959..19b6b77 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorHandler.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorHandler.java
@@ -100,36 +100,28 @@ public class PersistenceProcessorHandler implements WorkHandler<PersistenceProce
     // TODO Fix this method with the contents of PersistenceProcessor.flush() in master branch
     // TODO This is related to the changes in TestPersistenceProcessor.testCommitPersistenceWithHALeaseManager().
     // TODO Check also that test in the master branch
-    private void flush(Batch batch, long batchID) throws IOException {
+    private void flush(Batch batch, long batchID) {
 
-        long startFlushTimeInNs = System.nanoTime();
-
-        boolean areWeStillMaster = true;
-        if (!leaseManager.stillInLeasePeriod()) {
-            // The master TSO replica has changed, so we must inform the
-            // clients about it when sending the replies and avoid flushing
-            // the current batch of TXs
-            areWeStillMaster = false;
-            // We need also to clear the data in the buffer
-            writer.clearWriteBuffer();
-            LOG.trace("Replica {} lost mastership before flushing data", tsoHostAndPort);
-        } else {
+        if (batch.getNumEvents() > 0) {
+            commitSuicideIfNotMaster();
             try {
+                long startFlushTimeInNs = System.nanoTime();
                 writer.flush();
+                flushTimer.update(System.nanoTime() - startFlushTimeInNs);
+                batchSizeHistogram.update(batch.getNumEvents());
             } catch (IOException e) {
-                panicker.panic("Error persisting commit batch", e.getCause());
-            }
-            batchSizeHistogram.update(batch.getNumEvents());
-            if (!leaseManager.stillInLeasePeriod()) {
-                // If after flushing this TSO server is not the master
-                // replica we need inform the client about it
-                areWeStillMaster = false;
-                LOG.warn("Replica {} lost mastership after flushing data", tsoHostAndPort);
+                panicker.panic("Error persisting commit batch", e);
             }
+            commitSuicideIfNotMaster(); // TODO Here, we can return the client responses before committing suicide
+            batch.sendReply(replyProcessor, retryProc, batchID);
         }
-        flushTimer.update((System.nanoTime() - startFlushTimeInNs));
-        batch.sendReply(replyProcessor, retryProc, batchID, areWeStillMaster);
 
     }
 
-}
+    private void commitSuicideIfNotMaster() {
+        if (!leaseManager.stillInLeasePeriod()) {
+            panicker.panic("Replica " + tsoHostAndPort + " lost mastership whilst flushing data. Committing suicide");
+        }
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
index e82f0b1..e57cda3 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
@@ -24,6 +24,8 @@ import com.lmax.disruptor.EventFactory;
 import com.lmax.disruptor.RingBuffer;
 import com.lmax.disruptor.WorkerPool;
 import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import java.io.IOException;
@@ -33,6 +35,8 @@ import java.util.concurrent.Executors;
 
 class PersistenceProcessorImpl implements PersistenceProcessor {
 
+    private static final Logger LOG = LoggerFactory.getLogger(PersistenceProcessorImpl.class);
+
     private static final long INITIAL_LWM_VALUE = -1L;
 
     private final ReplyProcessor replyProcessor;
@@ -78,16 +82,6 @@ class PersistenceProcessorImpl implements PersistenceProcessor {
     }
 
     @Override
-    public void reset() throws InterruptedException {
-
-        batchIDCnt = 0L;
-        batchPool.reset();
-        batch = batchPool.getNextEmptyBatch();
-        replyProcessor.reset();
-
-    }
-
-    @Override
     public void persistFlush() throws InterruptedException {
 
         if (batch.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
index 0a3a00d..37a9dd9 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
@@ -38,13 +38,11 @@ interface ReplyProcessor {
      * @param channel
      *            the communication channed with the client
      */
-    void batchResponse(Batch batch, long batchID, boolean makeHeuristicDecision);
+    void batchResponse(Batch batch, long batchID);
 
     void addAbort(Batch batch, long startTimestamp, Channel c, MonitoringContext context);
 
     void addCommit(Batch batch, long startTimestamp, long commitTimestamp, Channel c, MonitoringContext context);
 
-    void reset();
-
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
index 55162c7..85f465c 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
@@ -88,10 +88,6 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
 
     }
 
-    public void reset() {
-        nextIDToHandle.set(0);
-    }
-
     private void handleReplyBatchEvent(ReplyBatchEvent event) {
 
         String name;
@@ -103,7 +99,7 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
             case COMMIT:
                 name = "commitReplyProcessor";
                 localEvent.getMonCtx().timerStart(name);
-                handleCommitResponse(localEvent.getStartTimestamp(), localEvent.getCommitTimestamp(), localEvent.getChannel(), event.getMakeHeuristicDecision());
+                handleCommitResponse(localEvent.getStartTimestamp(), localEvent.getCommitTimestamp(), localEvent.getChannel());
                 localEvent.getMonCtx().timerStop(name);
                  break;
             case ABORT:
@@ -169,11 +165,11 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
     }
 
     @Override
-    public void batchResponse(Batch batch, long batchID, boolean makeHeuristicDecision) {
+    public void batchResponse(Batch batch, long batchID) {
 
         long seq = replyRing.next();
         ReplyBatchEvent e = replyRing.get(seq);
-        ReplyBatchEvent.makeReplyBatch(e, batch, batchID, makeHeuristicDecision);
+        ReplyBatchEvent.makeReplyBatch(e, batch, batchID);
         replyRing.publish(seq);
 
     }
@@ -182,7 +178,7 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
     public void addAbort(Batch batch, long startTimestamp, Channel c, MonitoringContext context) {
 
         batch.addAbort(startTimestamp, true, c, context);
-        batchResponse(batch, NO_ORDER, false);
+        batchResponse(batch, NO_ORDER);
 
     }
 
@@ -190,19 +186,14 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
     public void addCommit(Batch batch, long startTimestamp, long commitTimestamp, Channel c, MonitoringContext context) {
 
         batch.addCommit(startTimestamp, commitTimestamp, c, context);
-        batchResponse(batch, NO_ORDER, false);
+        batchResponse(batch, NO_ORDER);
 
     }
 
-    private void handleCommitResponse(long startTimestamp, long commitTimestamp, Channel c,
-                                      boolean makeHeuristicDecision) {
+    private void handleCommitResponse(long startTimestamp, long commitTimestamp, Channel c) {
 
         TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
         TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
-        // TODO Remove heuristic decissions as is not in the protocol anymore
-        if (makeHeuristicDecision) { // If the commit is ambiguous is due to a new master TSO
-//            commitBuilder.setMakeHeuristicDecision(true);
-        }
         commitBuilder.setAborted(false)
                 .setStartTimestamp(startTimestamp)
                 .setCommitTimestamp(commitTimestamp);
@@ -242,12 +233,10 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
 
         private Batch batch;
         private long batchID;
-        private boolean makeHeuristicDecision;
 
-        static void makeReplyBatch(ReplyBatchEvent e, Batch batch, long batchID, boolean makeHeuristicDecision) {
+        static void makeReplyBatch(ReplyBatchEvent e, Batch batch, long batchID) {
             e.batch = batch;
             e.batchID = batchID;
-            e.makeHeuristicDecision = makeHeuristicDecision;
         }
 
         Batch getBatch() {
@@ -258,10 +247,6 @@ class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyBatchEv
             return batchID;
         }
 
-        boolean getMakeHeuristicDecision() {
-            return makeHeuristicDecision;
-        }
-
         final static EventFactory<ReplyBatchEvent> EVENT_FACTORY = new EventFactory<ReplyBatchEvent>() {
             public ReplyBatchEvent newInstance() {
                 return new ReplyBatchEvent();

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
index 709a529..2bbf1cf 100644
--- a/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
+++ b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
@@ -92,7 +92,7 @@ class RequestProcessorImpl implements EventHandler<RequestProcessorImpl.RequestE
         LOG.info("Initializing RequestProcessor...");
         this.lowWatermark = state.getLowWatermark();
         persistProc.persistLowWatermark(lowWatermark, new MonitoringContext(metrics));
-        persistProc.reset();
+        persistProc.persistFlush();
         LOG.info("RequestProcessor initialized with LWMs {} and Epoch {}", lowWatermark, state.getEpoch());
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java b/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
index 177e54d..08b38be 100644
--- a/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
@@ -118,12 +118,8 @@ public class TestBatch {
 //        assertFalse(batch.isFull(), "Batch shouldn't be full");
 //        assertEquals(batch.getNumEvents(), 0, "Num events should be 0");
 //=======
-        final boolean MASTER_INSTANCE = true;
-
-        final boolean SHOULD_MAKE_HEURISTIC_DECISION = true;
-        batch.sendReply(replyProcessor, retryProcessor, (-1), MASTER_INSTANCE);
-        verify(replyProcessor, timeout(100).times(1))
-               .batchResponse(batch, (-1), !SHOULD_MAKE_HEURISTIC_DECISION);
+        batch.sendReply(replyProcessor, retryProcessor, (-1));
+        verify(replyProcessor, timeout(100).times(1)).batchResponse(batch, (-1));
         assertTrue(batch.isFull(), "Batch shouldn't be empty");
     }
 
@@ -148,11 +144,9 @@ public class TestBatch {
         // Test that sending replies empties the batch also when the replica
         // is NOT master and calls the ambiguousCommitResponse() method on the
         // reply processor
-        final boolean MASTER_INSTANCE = true;
-        final boolean SHOULD_MAKE_HEURISTIC_DECISION = true;
-        batch.sendReply(replyProcessor, retryProcessor, (-1), !MASTER_INSTANCE);
+        batch.sendReply(replyProcessor, retryProcessor, (-1));
         verify(replyProcessor, timeout(100).times(1))
-               .batchResponse(batch, (-1), SHOULD_MAKE_HEURISTIC_DECISION);
+               .batchResponse(batch, (-1));
         assertTrue(batch.isFull(), "Batch should be full");
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/43261a3c/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java b/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
index b48cf8b..bc0b0f0 100644
--- a/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
@@ -20,6 +20,7 @@ package org.apache.omid.tso;
 import org.apache.omid.committable.CommitTable;
 import org.apache.omid.metrics.MetricsRegistry;
 import org.apache.omid.metrics.NullMetricsProvider;
+import org.jboss.netty.channel.Channel;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
@@ -34,22 +35,22 @@ import java.io.IOException;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
+// TODO Add timers
+// TODO Make visible currentBatch in PersistenceProcessorImpl to add proper verifications
 public class TestPersistenceProcessor {
 
     private static final Logger LOG = LoggerFactory.getLogger(TestPersistenceProcessor.class);
 
     @Mock
-    private Batch batch;
-    @Mock
     private CommitTable.Writer mockWriter;
     @Mock
     private CommitTable.Client mockClient;
@@ -69,7 +70,7 @@ public class TestPersistenceProcessor {
         MockitoAnnotations.initMocks(this);
 
         // Configure mock writer to flush successfully
-        doThrow(new IOException("Unable to write")).when(mockWriter).flush();
+//        doThrow(new IOException("Unable to write")).when(mockWriter).flush();
 
         // Configure null metrics provider
         metrics = new NullMetricsProvider();
@@ -86,6 +87,7 @@ public class TestPersistenceProcessor {
                 return mockClient;
             }
         };
+
     }
 
     @AfterMethod
@@ -96,12 +98,16 @@ public class TestPersistenceProcessor {
     @Test
     public void testCommitPersistenceWithMultiHandlers() throws Exception {
 
+        final int MAX_BATCH_SIZE = 4;
+        final int NUM_PERSIST_HANDLERS = 4;
+
         // Init a non-HA lease manager
         VoidLeaseManager leaseManager = spy(new VoidLeaseManager(mock(TSOChannelHandler.class),
-                mock(TSOStateManager.class)));
+                                                                 mock(TSOStateManager.class)));
 
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setPersistHandlerNum(4);
+        tsoConfig.setMaxBatchSize(MAX_BATCH_SIZE);
+        tsoConfig.setPersistHandlerNum(NUM_PERSIST_HANDLERS);
 
         PersistenceProcessorHandler[] handlers = new PersistenceProcessorHandler[tsoConfig.getPersistHandlerNum()];
         for (int i = 0; i < tsoConfig.getPersistHandlerNum(); i++) {
@@ -114,39 +120,45 @@ public class TestPersistenceProcessor {
         }
 
         // Component under test
+        BatchPool batchPool = spy(new BatchPool(tsoConfig));
+
         PersistenceProcessorImpl proc = new PersistenceProcessorImpl(tsoConfig,
-                                                                 new BatchPool(tsoConfig),
-                                                                 replyProcessor,
-                                                                 panicker,
-                                                                 handlers);
+                                                                     batchPool,
+                                                                     replyProcessor,
+                                                                     panicker,
+                                                                     handlers);
 
-        MonitoringContext monCtx = new MonitoringContext(metrics);
-        proc.batch = batch;
-        proc.persistCommit(1, 2, null, monCtx);
+        verify(batchPool, times(1)).getNextEmptyBatch();
+
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
-        proc.batch = batch;
-        proc.persistCommit(3, 4, null, monCtx);
+
+        proc.persistCommit(3, 4, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
-        proc.batch = batch;
-        proc.persistCommit(5, 6, null, monCtx);
+
+        proc.persistCommit(5, 6, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
-        proc.batch = batch;
-        proc.persistCommit(7, 8, null, monCtx);
+
+        proc.persistCommit(7, 8, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
-        verify(batch, timeout(1000).times(4)).sendReply(any(ReplyProcessor.class),
-                                                        any(RetryProcessor.class),
-                                                        any(Long.class), eq(true));
+
+        verify(batchPool, times(5)).getNextEmptyBatch(); // 5 Times: 1 in initialization + 4 when flushing above
+
     }
 
     @Test
-    public void testCommitPersistenceWithSingleHanlderInMultiHandlersEnvironment() throws Exception {
+    public void testCommitPersistenceWithSingleHandlerInMultiHandlersEnvironment() throws Exception {
+
+        final int MAX_BATCH_SIZE = 16;
+        final int NUM_PERSIST_HANDLERS = 4;
 
         // Init a non-HA lease manager
         VoidLeaseManager leaseManager = spy(new VoidLeaseManager(mock(TSOChannelHandler.class),
-                mock(TSOStateManager.class)));
+                                                                 mock(TSOStateManager.class)));
 
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setPersistHandlerNum(4);
+        tsoConfig.setMaxBatchSize(MAX_BATCH_SIZE);
+        tsoConfig.setPersistHandlerNum(NUM_PERSIST_HANDLERS);
 
         PersistenceProcessorHandler[] handlers = new PersistenceProcessorHandler[tsoConfig.getPersistHandlerNum()];
         for (int i = 0; i < tsoConfig.getPersistHandlerNum(); i++) {
@@ -160,51 +172,60 @@ public class TestPersistenceProcessor {
         }
 
         // Component under test
+        BatchPool batchPool = spy(new BatchPool(tsoConfig));
         PersistenceProcessorImpl proc = new PersistenceProcessorImpl(tsoConfig,
-                                                                 new BatchPool(tsoConfig),
-                                                                 replyProcessor,
-                                                                 panicker,
-                                                                 handlers);
+                                                                     batchPool,
+                                                                     replyProcessor,
+                                                                     panicker,
+                                                                     handlers);
 
-        MonitoringContext monCtx = new MonitoringContext(metrics);
-        proc.batch = batch;
-        proc.persistCommit(1, 2, null, monCtx);
+        verify(batchPool, times(1)).getNextEmptyBatch();
 
-        proc.persistCommit(3, 4, null, monCtx);
-        proc.persistCommit(5, 6, null, monCtx);
-        proc.persistCommit(7, 8, null, monCtx);
-        verify(batch, timeout(1000).times(0)).sendReply(any(ReplyProcessor.class),
-                any(RetryProcessor.class),
-                any(Long.class), eq(true));
+        // Fill one Batch completely
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistCommit(3, 4, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistCommit(5, 6, mock(Channel.class), mock(MonitoringContext.class));
+        verify(batchPool, times(1)).getNextEmptyBatch();
+        proc.persistCommit(7, 8, mock(Channel.class), mock(MonitoringContext.class)); // Should be full here
+        verify(batchPool, times(2)).getNextEmptyBatch();
+
+        // Test empty flush does not trigger response in getting a new batch
         proc.persistFlush();
-        verify(batch, timeout(1000).times(1)).sendReply(any(ReplyProcessor.class),
-                                                        any(RetryProcessor.class),
-                                                        any(Long.class), eq(true));
-        proc.batch = batch;
-        proc.persistCommit(1, 2, null, monCtx);
-        proc.persistCommit(3, 4, null, monCtx);
-        proc.persistCommit(5, 6, null, monCtx);
-        proc.persistCommit(7, 8, null, monCtx);
-        verify(batch, timeout(1000).times(1)).sendReply(any(ReplyProcessor.class),
-                any(RetryProcessor.class),
-                any(Long.class), eq(true));
+        verify(batchPool, times(2)).getNextEmptyBatch();
+
+        // Fill a second Batch completely
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistCommit(3, 4, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistCommit(5, 6, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistCommit(7, 8, mock(Channel.class), mock(MonitoringContext.class)); // Should be full here
+        verify(batchPool, times(3)).getNextEmptyBatch();
+
+        // Start filling a new batch and flush it immediately
+        proc.persistCommit(9, 10, mock(Channel.class), mock(MonitoringContext.class));
+        verify(batchPool, times(3)).getNextEmptyBatch();
         proc.persistFlush();
-        verify(batch, timeout(1000).times(2)).sendReply(any(ReplyProcessor.class),
-                any(RetryProcessor.class),
-                any(Long.class), eq(true));
-        proc.batch = batch;
+        verify(batchPool, times(4)).getNextEmptyBatch();
 
         // Test empty flush does not trigger response
         proc.persistFlush();
-        verify(batch, timeout(1000).times(2)).sendReply(any(ReplyProcessor.class),
-                any(RetryProcessor.class),
-                any(Long.class), eq(true));
+        proc.persistFlush();
+        proc.persistFlush();
+        proc.persistFlush();
+        proc.persistFlush();
+        verify(batchPool, times(4)).getNextEmptyBatch();
+
     }
 
     @Test
     public void testCommitPersistenceWithNonHALeaseManager() throws Exception {
 
+        final int MAX_BATCH_SIZE = 4;
+        final int NUM_PERSIST_HANDLERS = 4;
+
         TSOServerConfig tsoConfig = new TSOServerConfig();
+        tsoConfig.setMaxBatchSize(MAX_BATCH_SIZE);
+        tsoConfig.setPersistHandlerNum(NUM_PERSIST_HANDLERS);
+        tsoConfig.setBatchPersistTimeoutInMs(100);
 
         // Init a non-HA lease manager
         VoidLeaseManager leaseManager = spy(new VoidLeaseManager(mock(TSOChannelHandler.class),
@@ -222,35 +243,41 @@ public class TestPersistenceProcessor {
         }
 
         // Component under test
+        BatchPool batchPool = spy(new BatchPool(tsoConfig));
         PersistenceProcessorImpl proc = new PersistenceProcessorImpl(tsoConfig,
-                                                                 new BatchPool(tsoConfig),
-                                                                 replyProcessor,
-                                                                 panicker,
-                                                                 handlers);
+                                                                     batchPool,
+                                                                     replyProcessor,
+                                                                     panicker,
+                                                                     handlers);
 
         // The non-ha lease manager always return true for
         // stillInLeasePeriod(), so verify the batch sends replies as master
-        MonitoringContext monCtx = new MonitoringContext(metrics);
-        proc.batch = batch;
-        proc.persistCommit(1, 2, null, monCtx);
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
         verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
-        verify(batch, timeout(1000).times(1)).sendReply(any(ReplyProcessor.class),
-                                                        any(RetryProcessor.class),
-                                                        any(Long.class), eq(true));
+        verify(batchPool, times(2)).getNextEmptyBatch();
+
     }
 
     @Test
     public void testCommitPersistenceWithHALeaseManagerMultiHandlers() throws Exception {
+        final int MAX_BATCH_SIZE = 4;
+        final int NUM_PERSIST_HANDLERS = 4;
+
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setPersistHandlerNum(4);
+        tsoConfig.setMaxBatchSize(MAX_BATCH_SIZE);
+        tsoConfig.setPersistHandlerNum(NUM_PERSIST_HANDLERS);
+        tsoConfig.setBatchPersistTimeoutInMs(100);
 
         testCommitPersistenceWithHALeaseManagerPerConfig(tsoConfig);
     }
 
     @Test
     public void testCommitPersistenceWithHALeaseManager() throws Exception {
-        testCommitPersistenceWithHALeaseManagerPerConfig(new TSOServerConfig());
+
+        TSOServerConfig tsoConfig = new TSOServerConfig();
+        testCommitPersistenceWithHALeaseManagerPerConfig(tsoConfig);
+
     }
 
     // TODO Recheck this tests comparing with previous master
@@ -267,49 +294,56 @@ public class TestPersistenceProcessor {
                                                           commitTable,
                                                           replyProcessor,
                                                           retryProcessor,
-                                                          panicker);
+                                                          new RuntimeExceptionPanicker());
         }
 
         // Component under test
+        BatchPool batchPool = spy(new BatchPool(tsoConfig));
         PersistenceProcessorImpl proc = new PersistenceProcessorImpl(tsoConfig,
-                                                                     new BatchPool(tsoConfig),
+                                                                     batchPool,
                                                                      replyProcessor,
                                                                      panicker,
                                                                      handlers);
 
         doReturn(true).when(leaseManager).stillInLeasePeriod();
         MonitoringContext monCtx = new MonitoringContext(metrics);
-        proc.batch = batch;
-        proc.persistCommit(1, 2, null, monCtx);
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
         verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
-        verify(batch, timeout(1000).times(1))
-                .sendReply(any(ReplyProcessor.class), any(RetryProcessor.class), any(Long.class), eq(true));
+        verify(batchPool, times(2)).getNextEmptyBatch();
 
-        // Test 2: Configure the lease manager to return true first and false later for stillInLeasePeriod, so verify
-        // the batch sends replies as non-master
+        // Test 2: Configure the lease manager to return true first and false later for stillInLeasePeriod
 
         // Reset stuff
         reset(leaseManager);
-        reset(batch);
-        proc.batch = batch;
         doReturn(true).doReturn(false).when(leaseManager).stillInLeasePeriod();
-        proc.persistCommit(1, 2, null, monCtx);
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
         verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
-        verify(batch, timeout(1000).times(1))
-                .sendReply(any(ReplyProcessor.class), any(RetryProcessor.class), any(Long.class), eq(false));
+        verify(batchPool, times(3)).getNextEmptyBatch();
+
+        // Test 3: Configure the lease manager to return false for stillInLeasePeriod
 
         // Reset stuff
         reset(leaseManager);
-        reset(batch);
-        proc.batch = batch;
         doReturn(false).when(leaseManager).stillInLeasePeriod();
-        proc.persistCommit(1, 2, null, monCtx);
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
+        proc.persistFlush();
+        verify(leaseManager, timeout(1000).times(1)).stillInLeasePeriod();
+        verify(batchPool, times(4)).getNextEmptyBatch();
+
+        // Test 4: Configure the lease manager to return true first and false later for stillInLeasePeriod and raise
+        // an exception when flush
+
+        // Reset stuff
+        reset(leaseManager);
+        // Configure mock writer to flush successfully
+        doThrow(new IOException("Unable to write")).when(mockWriter).flush();
+        doReturn(true).doReturn(false).when(leaseManager).stillInLeasePeriod();
+        proc.persistCommit(1, 2, mock(Channel.class), mock(MonitoringContext.class));
         proc.persistFlush();
         verify(leaseManager, timeout(1000).times(1)).stillInLeasePeriod();
-        verify(batch, timeout(1000).times(1))
-                .sendReply(any(ReplyProcessor.class), any(RetryProcessor.class), any(Long.class), eq(false));
+        verify(batchPool, times(5)).getNextEmptyBatch();
     }
 
     @Test
@@ -382,6 +416,7 @@ public class TestPersistenceProcessor {
         proc.persistCommit(1, 2, null, monCtx);
         proc.persistFlush();
         verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+
     }
 
 }