You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2021/05/13 04:22:27 UTC

[pulsar] branch master updated: [Transaction] Transaction pending ack persistent (#8881)

This is an automated email from the ASF dual-hosted git repository.

penghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 8f1f5ba  [Transaction] Transaction pending ack persistent (#8881)
8f1f5ba is described below

commit 8f1f5ba4fd8ecbab1f975623605400f5088c6e32
Author: congbo <39...@users.noreply.github.com>
AuthorDate: Thu May 13 12:21:38 2021 +0800

    [Transaction] Transaction pending ack persistent (#8881)
    
    ## Motivation
    in order to handle pending ack persistent.
    
    ## implement
    1. add the transaction pending ack store, it will handle the pending ack metadata store.
    2. when the sub unload, we will replay the pendingAckHandle.
    3. we use one manageLedger to store the pending ack metadata by one sub , and replay by this managedLedger open cursor.
    4. when we commit or abort the transaction, we will append the marker to the pendingAckStore then we will modify state memory in pendingAckHandle
    4. we also modify the in memory state when append fail, because we don't know the persistent state, when we replay it, it will produce the wrong operation. so we append fail, we should wait tc time out or client abort this transaction.
    5. when we append pending ack log, we will compare the the log position store the biggest topic position is bigger than persistent topic markDeletePosition. if it is smaller, will delete the position.
    ### Verifying this change
    Add the tests for it
---
 .../apache/bookkeeper/mledger/ManagedCursor.java   |   7 +
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |  31 +-
 .../mledger/impl/ManagedCursorConcurrencyTest.java |   2 +-
 .../mledger/impl/ManagedCursorContainerTest.java   |   5 +
 .../apache/pulsar/broker/ServiceConfiguration.java |  21 +-
 pulsar-broker/pom.xml                              |  10 +-
 .../org/apache/pulsar/broker/PulsarService.java    |  28 +-
 .../broker/TransactionMetadataStoreService.java    |   8 +-
 .../pulsar/broker/service/AbstractTopic.java       |   9 +-
 .../org/apache/pulsar/broker/service/Consumer.java |   2 +-
 .../apache/pulsar/broker/service/ServerCnx.java    |   2 +-
 .../apache/pulsar/broker/service/Subscription.java |   2 +-
 .../nonpersistent/NonPersistentSubscription.java   |  16 +-
 .../service/nonpersistent/NonPersistentTopic.java  |  40 +-
 .../service/persistent/PersistentSubscription.java | 166 +++---
 .../broker/service/persistent/PersistentTopic.java |  39 +-
 .../buffer/impl/TopicTransactionBuffer.java        |   5 +-
 .../buffer/impl/TransactionBufferHandlerImpl.java  |   2 +-
 .../transaction/pendingack/PendingAckHandle.java   |  20 +-
 .../pendingack/PendingAckReplyCallBack.java        |  39 ++
 .../transaction/pendingack/PendingAckStore.java    |  85 +++
 .../TransactionPendingAckStoreProvider.java        |  62 ++
 ...ransactionPendingAckStoreProviderException.java |  32 +
 .../pendingack/exceptions/package-info.java        |  23 +
 .../pendingack/impl/InMemoryPendingAckStore.java   |  66 ++
 .../impl/InMemoryPendingAckStoreProvider.java      |  32 +
 .../pendingack/impl/MLPendingAckReplyCallBack.java | 108 ++++
 .../pendingack/impl/MLPendingAckStore.java         | 396 ++++++++++++
 .../pendingack/impl/MLPendingAckStoreProvider.java |  86 +++
 .../pendingack/impl/PendingAckHandleDisabled.java  |  16 +-
 .../pendingack/impl/PendingAckHandleImpl.java      | 664 +++++++++++++++------
 .../pendingack/impl/PendingAckHandleState.java     |  70 +++
 .../src/main/proto/TransactionPendingAck.proto     |  45 ++
 .../pulsar/broker/service/PersistentTopicTest.java |  58 +-
 .../service/TransactionMarkerDeleteTest.java       |   4 +-
 .../persistent/PersistentSubscriptionTest.java     |  62 +-
 .../broker/transaction/TransactionConsumeTest.java |   6 +
 .../broker/transaction/TransactionProduceTest.java |   4 +
 .../pendingack/PendingAckInMemoryDeleteTest.java   |  86 +--
 .../pendingack/PendingAckPersistentTest.java       | 301 ++++++++++
 .../client/impl/TransactionEndToEndTest.java       |   8 +-
 .../integration/transaction/TransactionTest.java   |   3 +
 42 files changed, 2281 insertions(+), 390 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index be898f3..03419de 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -393,6 +393,13 @@ public interface ManagedCursor {
     Position getMarkDeletedPosition();
 
     /**
+     * Get the persistent newest mark deleted position on this cursor.
+     *
+     * @return the persistent mark deleted position
+     */
+    Position getPersistentMarkDeletedPosition();
+
+    /**
      * Rewind the cursor to the mark deleted position to replay all the already read but not yet mark deleted messages.
      *
      * <p/>The next message to be read is the one after the current mark deleted message.
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index dc89dbb..0d249a1 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -116,6 +116,9 @@ public class ManagedCursorImpl implements ManagedCursor {
 
     protected volatile PositionImpl markDeletePosition;
 
+    // this position is have persistent mark delete position
+    protected volatile PositionImpl persistentMarkDeletePosition;
+
     protected static final AtomicReferenceFieldUpdater<ManagedCursorImpl, PositionImpl> READ_POSITION_UPDATER =
             AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, "readPosition");
     protected volatile PositionImpl readPosition;
@@ -500,6 +503,7 @@ public class ManagedCursorImpl implements ManagedCursor {
 
         messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition()));
         markDeletePosition = position;
+        persistentMarkDeletePosition = position;
         readPosition = ledger.getNextValidPosition(position);
         lastMarkDeleteEntry = new MarkDeleteEntry(markDeletePosition, properties, null, null);
         // assign cursor-ledger so, it can be deleted when new ledger will be switched
@@ -576,7 +580,8 @@ public class ManagedCursorImpl implements ManagedCursor {
                                  Object ctx, PositionImpl maxPosition) {
         checkArgument(numberOfEntriesToRead > 0);
         if (isClosed()) {
-            callback.readEntriesFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
+            callback.readEntriesFailed(new ManagedLedgerException
+                    .CursorAlreadyClosedException("Cursor was already closed"), ctx);
             return;
         }
 
@@ -628,7 +633,8 @@ public class ManagedCursorImpl implements ManagedCursor {
             Object ctx) {
         checkArgument(n > 0);
         if (isClosed()) {
-            callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
+            callback.readEntryFailed(new ManagedLedgerException
+                    .CursorAlreadyClosedException("Cursor was already closed"), ctx);
             return;
         }
 
@@ -781,7 +787,7 @@ public class ManagedCursorImpl implements ManagedCursor {
         }
     }
 
-    private boolean isClosed() {
+    public boolean isClosed() {
         return state == State.Closed || state == State.Closing;
     }
 
@@ -1597,7 +1603,8 @@ public class ManagedCursorImpl implements ManagedCursor {
         checkArgument(position instanceof PositionImpl);
 
         if (isClosed()) {
-            callback.markDeleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
+            callback.markDeleteFailed(new ManagedLedgerException
+                    .CursorAlreadyClosedException("Cursor was already closed"), ctx);
             return;
         }
 
@@ -1673,7 +1680,8 @@ public class ManagedCursorImpl implements ManagedCursor {
             // The state might have changed while we were waiting on the queue mutex
             switch (STATE_UPDATER.get(this)) {
             case Closed:
-                callback.markDeleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
+                callback.markDeleteFailed(new ManagedLedgerException
+                        .CursorAlreadyClosedException("Cursor was already closed"), ctx);
                 return;
 
             case NoLedger:
@@ -1729,6 +1737,11 @@ public class ManagedCursorImpl implements ManagedCursor {
                         subMap.values().forEach(BitSetRecyclable::recycle);
                         subMap.clear();
                     }
+                    if (persistentMarkDeletePosition == null
+                            || mdEntry.newPosition.compareTo(persistentMarkDeletePosition) > 0) {
+                        persistentMarkDeletePosition = mdEntry.newPosition;
+                    }
+
                 } finally {
                     lock.writeLock().unlock();
                 }
@@ -1835,7 +1848,8 @@ public class ManagedCursorImpl implements ManagedCursor {
     @Override
     public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
         if (isClosed()) {
-            callback.deleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx);
+            callback.deleteFailed(new ManagedLedgerException
+                    .CursorAlreadyClosedException("Cursor was already closed"), ctx);
             return;
         }
 
@@ -2065,6 +2079,11 @@ public class ManagedCursorImpl implements ManagedCursor {
     }
 
     @Override
+    public Position getPersistentMarkDeletedPosition() {
+        return this.persistentMarkDeletePosition;
+    }
+
+    @Override
     public void rewind() {
         lock.writeLock().lock();
         try {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
index 856a7e8..0f6386d 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
@@ -161,7 +161,7 @@ public class ManagedCursorConcurrencyTest extends MockedBookKeeperTestCase {
                         Thread.sleep(1);
                     }
                 } catch (ManagedLedgerException e) {
-                    if (!e.getMessage().equals("Cursor was already closed")) {
+                    if (!(e instanceof ManagedLedgerException.CursorAlreadyClosedException)) {
                         gotException.set(true);
                     }
 
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java
index baf1c9f..5490278 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java
@@ -131,6 +131,11 @@ public class ManagedCursorContainerTest {
         }
 
         @Override
+        public Position getPersistentMarkDeletedPosition() {
+            return position;
+        }
+
+        @Override
         public String getName() {
             return name;
         }
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 8615e58..a09077f 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -221,13 +221,6 @@ public class ServiceConfiguration implements PulsarConfiguration {
     )
     private int numCacheExecutorThreadPoolSize = 10;
 
-    @FieldContext(
-            category = CATEGORY_SERVER,
-            doc = "Number of threads to use for pulsar broker service."
-                    + " The executor in thread pool will do transaction recover"
-    )
-    private int numTransactionExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors();
-
     @FieldContext(category = CATEGORY_SERVER, doc = "Max concurrent web requests")
     private int maxConcurrentHttpRequests = 1024;
 
@@ -2041,6 +2034,20 @@ public class ServiceConfiguration implements PulsarConfiguration {
 
     @FieldContext(
             category = CATEGORY_TRANSACTION,
+            doc = "Class name for transaction pending ack store provider"
+    )
+    private String transactionPendingAckStoreProviderClassName =
+            "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider";
+
+    @FieldContext(
+            category = CATEGORY_TRANSACTION,
+            doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer."
+                    + "Default is 5"
+    )
+    private int numTransactionReplayThreadPoolSize = Runtime.getRuntime().availableProcessors();
+
+    @FieldContext(
+            category = CATEGORY_TRANSACTION,
             doc = "Transaction buffer take snapshot transaction count"
     )
     private int transactionBufferSnapshotMaxTransactionCount = 1000;
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 5e924ab..b752f42 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -477,7 +477,10 @@
         <configuration>
           <protocArtifact>com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier}</protocArtifact>
           <checkStaleness>true</checkStaleness>
-          <excludes>**/ResourceUsage.proto</excludes>
+          <excludes>
+            <exclude>**/ResourceUsage.proto</exclude>
+            <exclude>**/TransactionPendingAck.proto</exclude>
+          </excludes>
         </configuration>
         <executions>
           <execution>
@@ -495,7 +498,10 @@
         <artifactId>lightproto-maven-plugin</artifactId>
         <version>${lightproto-maven-plugin.version}</version>
         <configuration>
-          <sources>${project.basedir}/src/main/proto/ResourceUsage.proto</sources>
+          <sources>
+            <source>${project.basedir}/src/main/proto/TransactionPendingAck.proto</source>
+            <source>${project.basedir}/src/main/proto/ResourceUsage.proto</source>
+          </sources>
           <targetSourcesSubDir>generated-sources/lightproto/java</targetSourcesSubDir>
           <targetTestSourcesSubDir>generated-sources/lightproto/java</targetTestSourcesSubDir>
         </configuration>
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 51fba11..6ed22f5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -106,6 +106,7 @@ import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider;
 import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
 import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider;
 import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
 import org.apache.pulsar.broker.validator.MultipleListenerValidator;
 import org.apache.pulsar.broker.web.WebService;
 import org.apache.pulsar.client.admin.PulsarAdmin;
@@ -223,7 +224,6 @@ public class PulsarService implements AutoCloseable {
     private TransactionMetadataStoreService transactionMetadataStoreService;
     private TransactionBufferProvider transactionBufferProvider;
     private TransactionBufferClient transactionBufferClient;
-    private ScheduledExecutorService transactionExecutor;
     private HashedWheelTimer transactionTimer;
 
     private BrokerInterceptor brokerInterceptor;
@@ -240,6 +240,9 @@ public class PulsarService implements AutoCloseable {
     private MetadataStoreExtended configurationMetadataStore;
     private PulsarResources pulsarResources;
 
+    private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider;
+    private final ScheduledExecutorService transactionReplayExecutor;
+
     public enum State {
         Init, Started, Closing, Closed
     }
@@ -287,6 +290,14 @@ public class PulsarService implements AutoCloseable {
                 new DefaultThreadFactory("pulsar"));
         this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(),
                 new DefaultThreadFactory("zk-cache-callback"));
+
+        if (config.isTransactionCoordinatorEnabled()) {
+            this.transactionReplayExecutor = Executors.newScheduledThreadPool(
+                    config.getNumTransactionReplayThreadPoolSize(),
+                    new DefaultThreadFactory("transaction-replay"));
+        } else {
+            this.transactionReplayExecutor = null;
+        }
         }
 
     public MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException {
@@ -445,8 +456,6 @@ public class PulsarService implements AutoCloseable {
                 transactionBufferClient.close();
             }
 
-            executorServicesShutdown.shutdown(transactionExecutor);
-
             if (coordinationService != null) {
                 coordinationService.close();
             }
@@ -458,6 +467,10 @@ public class PulsarService implements AutoCloseable {
                 configurationMetadataStore.close();
             }
 
+            if (transactionReplayExecutor != null) {
+                transactionReplayExecutor.shutdown();
+            }
+
             // add timeout handling for closing executors
             asyncCloseFutures.add(executorServicesShutdown.handle());
 
@@ -729,9 +742,6 @@ public class PulsarService implements AutoCloseable {
 
             // Register pulsar system namespaces and start transaction meta store service
             if (config.isTransactionCoordinatorEnabled()) {
-                this.transactionExecutor = Executors.newScheduledThreadPool(
-                        config.getNumTransactionExecutorThreadPoolSize(),
-                        new DefaultThreadFactory("pulsar-transaction"));
                 this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient());
                 this.transactionTimer =
                         new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer"));
@@ -744,6 +754,8 @@ public class PulsarService implements AutoCloseable {
 
                 transactionBufferProvider = TransactionBufferProvider
                         .newProvider(config.getTransactionBufferProviderClassName());
+                transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider
+                        .newProvider(config.getTransactionPendingAckStoreProviderClassName());
             }
 
             this.metricsGenerator = new MetricsGenerator(this);
@@ -1170,6 +1182,10 @@ public class PulsarService implements AutoCloseable {
         return cacheExecutor;
     }
 
+    public ScheduledExecutorService getTransactionReplayExecutor() {
+        return transactionReplayExecutor;
+    }
+
     public ScheduledExecutorService getLoadManagerExecutor() {
         return loadManagerExecutor;
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
index 552e9a8..450dcbb 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java
@@ -399,17 +399,13 @@ public class TransactionMetadataStoreService {
     }
 
     private static boolean isRetryableException(Throwable e) {
-        if (e instanceof TransactionMetadataStoreStateException
+        return e instanceof TransactionMetadataStoreStateException
                 || e instanceof RequestTimeoutException
                 || e instanceof ManagedLedgerException
                 || e instanceof BrokerPersistenceException
                 || e instanceof LookupException
                 || e instanceof ReachMaxPendingOpsException
-                || e instanceof ConnectException) {
-            return true;
-        } else {
-            return false;
-        }
+                || e instanceof ConnectException;
     }
 
     private CompletableFuture<Void> endTxnInTransactionMetadataStore(TxnID txnID, int txnAction) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 23178b3..fafd098 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -253,19 +253,18 @@ public abstract class AbstractTopic implements Topic {
         return count;
     }
 
-    protected void addConsumerToSubscription(Subscription subscription, Consumer consumer)
-            throws BrokerServiceException {
+    protected CompletableFuture<Void> addConsumerToSubscription(Subscription subscription, Consumer consumer) {
         if (isConsumersExceededOnTopic()) {
             log.warn("[{}] Attempting to add consumer to topic which reached max consumers limit", topic);
-            throw new ConsumerBusyException("Topic reached max consumers limit");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Topic reached max consumers limit"));
         }
 
         if (isSameAddressConsumersExceededOnTopic(consumer)) {
             log.warn("[{}] Attempting to add consumer to topic which reached max same address consumers limit", topic);
-            throw new ConsumerBusyException("Topic reached max same address consumers limit");
+            return FutureUtil.failedFuture(new ConsumerBusyException("Topic reached max same address consumers limit"));
         }
 
-        subscription.addConsumer(consumer);
+        return subscription.addConsumer(consumer);
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index 118d11b..f905cb5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -128,7 +128,7 @@ public class Consumer {
                     int priorityLevel, String consumerName,
                     int maxUnackedMessages, TransportCnx cnx, String appId,
                     Map<String, String> metadata, boolean readCompacted, InitialPosition subscriptionInitialPosition,
-                    KeySharedMeta keySharedMeta) throws BrokerServiceException {
+                    KeySharedMeta keySharedMeta) {
 
         this.subscription = subscription;
         this.subType = subType;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index c03e6fc..8320fc8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -1912,7 +1912,7 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
         if (topicFuture != null) {
             topicFuture.whenComplete((optionalTopic, t) -> {
                 if (!optionalTopic.isPresent()) {
-                    log.error("handleEndTxnOnPartition faile ! The topic {} does not exist in broker, "
+                    log.error("handleEndTxnOnPartition fail ! The topic {} does not exist in broker, "
                             + "txnId: [{}], txnAction: [{}]", topic, txnID, TxnAction.valueOf(txnAction));
                     ctx.writeAndFlush(Commands.newEndTxnOnPartitionResponse(
                             requestId, ServerError.ServiceNotReady,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java
index 997831e..555b10d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java
@@ -37,7 +37,7 @@ public interface Subscription {
 
     String getName();
 
-    void addConsumer(Consumer consumer) throws BrokerServiceException;
+    CompletableFuture<Void> addConsumer(Consumer consumer);
 
     default void removeConsumer(Consumer consumer) throws BrokerServiceException {
         removeConsumer(consumer, false);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
index 7cce496..0fb05c6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java
@@ -46,6 +46,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ConsumerStats;
 import org.apache.pulsar.common.policies.data.NonPersistentSubscriptionStats;
+import org.apache.pulsar.common.util.FutureUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -97,11 +98,11 @@ public class NonPersistentSubscription implements Subscription {
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
+    public synchronized CompletableFuture<Void> addConsumer(Consumer consumer) {
         updateLastActive();
         if (IS_FENCED_UPDATER.get(this) == TRUE) {
             log.warn("Attempting to add consumer {} on a fenced subscription", consumer);
-            throw new SubscriptionFencedException("Subscription is fenced");
+            return FutureUtil.failedFuture(new SubscriptionFencedException("Subscription is fenced"));
         }
 
         if (dispatcher == null || !dispatcher.isConsumerConnected()) {
@@ -160,7 +161,7 @@ public class NonPersistentSubscription implements Subscription {
                 }
                 break;
             default:
-                throw new ServerMetadataException("Unsupported subscription type");
+                return FutureUtil.failedFuture(new ServerMetadataException("Unsupported subscription type"));
             }
 
             if (previousDispatcher != null) {
@@ -173,11 +174,16 @@ public class NonPersistentSubscription implements Subscription {
             }
         } else {
             if (consumer.subType() != dispatcher.getType()) {
-                throw new SubscriptionBusyException("Subscription is of different type");
+                return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription is of different type"));
             }
         }
 
-        dispatcher.addConsumer(consumer);
+        try {
+            dispatcher.addConsumer(consumer);
+            return CompletableFuture.completedFuture(null);
+        } catch (BrokerServiceException brokerServiceException) {
+            return FutureUtil.failedFuture(brokerServiceException);
+        }
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index 353967a..5a61608 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -265,13 +265,24 @@ public class NonPersistentTopic extends AbstractTopic implements Topic {
 
         NonPersistentSubscription subscription = subscriptions.computeIfAbsent(subscriptionName,
                 name -> new NonPersistentSubscription(this, subscriptionName));
-
-        try {
-            Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, 0,
-                    cnx, cnx.getAuthRole(), metadata, readCompacted, initialPosition, keySharedMeta);
-            addConsumerToSubscription(subscription, consumer);
+        Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, 0,
+                cnx, cnx.getAuthRole(), metadata, readCompacted, initialPosition, keySharedMeta);
+        addConsumerToSubscription(subscription, consumer).thenRun(() -> {
             if (!cnx.isActive()) {
-                consumer.close();
+                try {
+                    consumer.close();
+                } catch (BrokerServiceException e) {
+                    if (e instanceof ConsumerBusyException) {
+                        log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId,
+                                consumerName);
+                    } else if (e instanceof SubscriptionBusyException) {
+                        log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
+                    }
+
+                    decrementUsageCount();
+                    future.completeExceptionally(e);
+                    return;
+                }
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName,
                             consumer.consumerName(), currentUsageCount());
@@ -282,17 +293,19 @@ public class NonPersistentTopic extends AbstractTopic implements Topic {
                 log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId);
                 future.complete(consumer);
             }
-        } catch (BrokerServiceException e) {
-            if (e instanceof ConsumerBusyException) {
+        }).exceptionally(e -> {
+            Throwable throwable = e.getCause();
+            if (throwable instanceof ConsumerBusyException) {
                 log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId,
                         consumerName);
-            } else if (e instanceof SubscriptionBusyException) {
+            } else if (throwable instanceof SubscriptionBusyException) {
                 log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
             }
 
             decrementUsageCount();
-            future.completeExceptionally(e);
-        }
+            future.completeExceptionally(throwable);
+            return null;
+        });
 
         return future;
     }
@@ -843,11 +856,12 @@ public class NonPersistentTopic extends AbstractTopic implements Topic {
                     stopReplProducers().thenCompose(v -> delete(true, false, true))
                             .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic))
                             .exceptionally(e -> {
-                                if (e.getCause() instanceof TopicBusyException) {
+                                Throwable throwable = e.getCause();
+                                if (throwable instanceof TopicBusyException) {
                                     // topic became active again
                                     if (log.isDebugEnabled()) {
                                         log.debug("[{}] Did not delete busy topic: {}", topic,
-                                                e.getCause().getMessage());
+                                                throwable.getMessage());
                                     }
                                     replicators.forEach((region, replicator) -> replicator.startProducer());
                                 } else {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 66c666b..320fdc6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import java.util.Collections;
@@ -56,6 +57,7 @@ import org.apache.pulsar.broker.service.Dispatcher;
 import org.apache.pulsar.broker.service.Subscription;
 import org.apache.pulsar.broker.service.Topic;
 import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle;
+import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore;
 import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleDisabled;
 import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl;
 import org.apache.pulsar.client.api.transaction.TxnID;
@@ -71,6 +73,7 @@ import org.apache.pulsar.common.policies.data.SubscriptionStats;
 import org.apache.pulsar.common.protocol.Commands;
 import org.apache.pulsar.common.protocol.Markers;
 import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -130,7 +133,11 @@ public class PersistentSubscription implements Subscription {
         this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString();
         this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this);
         this.setReplicated(replicated);
-        if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled()) {
+        if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled()
+                && !checkTopicIsEventsNames(topicName)
+                && !topicName.startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName())
+                && !topicName.startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)
+                && !topicName.endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) {
             this.pendingAckHandle = new PendingAckHandleImpl(this);
         } else {
             this.pendingAckHandle = new PendingAckHandleDisabled();
@@ -172,78 +179,93 @@ public class PersistentSubscription implements Subscription {
     }
 
     @Override
-    public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
-        cursor.updateLastActive();
-        if (IS_FENCED_UPDATER.get(this) == TRUE) {
-            log.warn("Attempting to add consumer {} on a fenced subscription", consumer);
-            throw new SubscriptionFencedException("Subscription is fenced");
-        }
-
-        if (dispatcher == null || !dispatcher.isConsumerConnected()) {
-            Dispatcher previousDispatcher = null;
-            boolean useStreamingDispatcher = topic.getBrokerService().getPulsar()
-                                                    .getConfiguration().isStreamingDispatch();
-            switch (consumer.subType()) {
-            case Exclusive:
-                if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) {
-                    previousDispatcher = dispatcher;
-                    dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer(cursor,
-                            SubType.Exclusive, 0, topic, this) :
-                            new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Exclusive, 0,
-                                    topic, this);
-                }
-                break;
-            case Shared:
-                if (dispatcher == null || dispatcher.getType() != SubType.Shared) {
-                    previousDispatcher = dispatcher;
-                    dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherMultipleConsumers(topic,
-                            cursor, this) : new PersistentDispatcherMultipleConsumers(topic,
-                            cursor, this);
-                }
-                break;
-            case Failover:
-                int partitionIndex = TopicName.getPartitionIndex(topicName);
-                if (partitionIndex < 0) {
-                    // For non partition topics, use a negative index so dispatcher won't sort consumers before picking
-                    // an active consumer for the topic.
-                    partitionIndex = -1;
+    public CompletableFuture<Void> addConsumer(Consumer consumer) {
+        return pendingAckHandle.pendingAckHandleFuture().thenCompose(future -> {
+            synchronized (PersistentSubscription.this) {
+                cursor.updateLastActive();
+                if (IS_FENCED_UPDATER.get(this) == TRUE) {
+                    log.warn("Attempting to add consumer {} on a fenced subscription", consumer);
+                    return FutureUtil.failedFuture(new SubscriptionFencedException("Subscription is fenced"));
                 }
 
-                if (dispatcher == null || dispatcher.getType() != SubType.Failover) {
-                    previousDispatcher = dispatcher;
-                    dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer(cursor,
-                            SubType.Failover, partitionIndex, topic, this) :
-                            new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover,
-                                    partitionIndex, topic, this);
-                }
-                break;
-            case Key_Shared:
-                if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) {
-                    previousDispatcher = dispatcher;
-                    KeySharedMeta ksm = consumer.getKeySharedMeta();
-                    dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this,
-                            topic.getBrokerService().getPulsar().getConfiguration(), ksm);
+                if (dispatcher == null || !dispatcher.isConsumerConnected()) {
+                    Dispatcher previousDispatcher = null;
+                    boolean useStreamingDispatcher = topic.getBrokerService().getPulsar()
+                            .getConfiguration().isStreamingDispatch();
+                    switch (consumer.subType()) {
+                        case Exclusive:
+                            if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) {
+                                previousDispatcher = dispatcher;
+                                dispatcher = useStreamingDispatcher
+                                        ? new PersistentStreamingDispatcherSingleActiveConsumer(
+                                                cursor, SubType.Exclusive, 0, topic, this)
+                                        : new PersistentDispatcherSingleActiveConsumer(
+                                                cursor, SubType.Exclusive, 0, topic, this);
+                            }
+                            break;
+                        case Shared:
+                            if (dispatcher == null || dispatcher.getType() != SubType.Shared) {
+                                previousDispatcher = dispatcher;
+                                dispatcher = useStreamingDispatcher
+                                        ? new PersistentStreamingDispatcherMultipleConsumers(
+                                                topic, cursor, this)
+                                        : new PersistentDispatcherMultipleConsumers(topic, cursor, this);
+                            }
+                            break;
+                        case Failover:
+                            int partitionIndex = TopicName.getPartitionIndex(topicName);
+                            if (partitionIndex < 0) {
+                                // For non partition topics, use a negative index so
+                                // dispatcher won't sort consumers before picking
+                                // an active consumer for the topic.
+                                partitionIndex = -1;
+                            }
+
+                            if (dispatcher == null || dispatcher.getType() != SubType.Failover) {
+                                previousDispatcher = dispatcher;
+                                dispatcher = useStreamingDispatcher
+                                        ? new PersistentStreamingDispatcherSingleActiveConsumer(
+                                                cursor, SubType.Failover, partitionIndex, topic, this) :
+                                        new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover,
+                                                partitionIndex, topic, this);
+                            }
+                            break;
+                        case Key_Shared:
+                            if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) {
+                                previousDispatcher = dispatcher;
+                                KeySharedMeta ksm = consumer.getKeySharedMeta();
+                                dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this,
+                                        topic.getBrokerService().getPulsar().getConfiguration(), ksm);
+                            }
+                            break;
+                        default:
+                            return FutureUtil.failedFuture(
+                                    new ServerMetadataException("Unsupported subscription type"));
+                    }
+
+                    if (previousDispatcher != null) {
+                        previousDispatcher.close().thenRun(() -> {
+                            log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName);
+                        }).exceptionally(ex -> {
+                            log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex);
+                            return null;
+                        });
+                    }
+                } else {
+                    if (consumer.subType() != dispatcher.getType()) {
+                        return FutureUtil.failedFuture(
+                                new SubscriptionBusyException("Subscription is of different type"));
+                    }
                 }
-                break;
-            default:
-                throw new ServerMetadataException("Unsupported subscription type");
-            }
 
-            if (previousDispatcher != null) {
-                previousDispatcher.close().thenRun(() -> {
-                    log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName);
-                }).exceptionally(ex -> {
-                    log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex);
-                    return null;
-                });
-            }
-        } else {
-            if (consumer.subType() != dispatcher.getType()) {
-                throw new SubscriptionBusyException("Subscription is of different type");
+                try {
+                    dispatcher.addConsumer(consumer);
+                    return CompletableFuture.completedFuture(null);
+                } catch (BrokerServiceException brokerServiceException) {
+                    return FutureUtil.failedFuture(brokerServiceException);
+                }
             }
-        }
-
-        dispatcher.addConsumer(consumer);
+        });
     }
 
     @Override
@@ -750,11 +772,11 @@ public class PersistentSubscription implements Subscription {
             if (dispatcher != null && dispatcher.isConsumerConnected()) {
                 return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers"));
             }
-            IS_FENCED_UPDATER.set(this, TRUE);
-            log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor);
+            return this.pendingAckHandle.close().thenAccept(v -> {
+                IS_FENCED_UPDATER.set(this, TRUE);
+                log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor);
+            });
         }
-
-        return CompletableFuture.completedFuture(null);
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 28c541a..ee5d0d9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -103,6 +103,7 @@ import org.apache.pulsar.broker.stats.NamespaceStats;
 import org.apache.pulsar.broker.stats.ReplicationMetrics;
 import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer;
 import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable;
+import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore;
 import org.apache.pulsar.client.admin.LongRunningProcessStatus;
 import org.apache.pulsar.client.admin.OffloadProcessStatus;
 import org.apache.pulsar.client.api.MessageId;
@@ -308,7 +309,8 @@ public class PersistentTopic extends AbstractTopic
         if (brokerService.getPulsar().getConfiguration().isTransactionCoordinatorEnabled()
                 && !checkTopicIsEventsNames(topic)
                 && !topicName.getEncodedLocalName().startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName())
-                && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)) {
+                && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)
+                && !topicName.getEncodedLocalName().endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) {
             this.transactionBuffer = brokerService.getPulsar()
                     .getTransactionBufferProvider().newTransactionBuffer(this, transactionCompletableFuture);
         } else {
@@ -724,16 +726,26 @@ public class PersistentTopic extends AbstractTopic
                 : 0;
 
         subscriptionFuture.thenAccept(subscription -> {
-            try {
-                Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName,
-                        maxUnackedMessages, cnx, cnx.getAuthRole(), metadata,
-                        readCompacted, initialPosition, keySharedMeta);
-                addConsumerToSubscription(subscription, consumer);
-
+            Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName,
+                    maxUnackedMessages, cnx, cnx.getAuthRole(), metadata,
+                    readCompacted, initialPosition, keySharedMeta);
+            addConsumerToSubscription(subscription, consumer).thenAccept(v -> {
                 checkBackloggedCursors();
-
                 if (!cnx.isActive()) {
-                    consumer.close();
+                    try {
+                        consumer.close();
+                    } catch (BrokerServiceException e) {
+                        if (e instanceof ConsumerBusyException) {
+                            log.warn("[{}][{}] Consumer {} {} already connected",
+                                    topic, subscriptionName, consumerId, consumerName);
+                        } else if (e instanceof SubscriptionBusyException) {
+                            log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
+                        }
+
+                        decrementUsageCount();
+                        future.completeExceptionally(e);
+                        return;
+                    }
                     if (log.isDebugEnabled()) {
                         log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName,
                                 consumer.consumerName(), currentUsageCount());
@@ -747,17 +759,18 @@ public class PersistentTopic extends AbstractTopic
                     log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId);
                     future.complete(consumer);
                 }
-            } catch (BrokerServiceException e) {
-                if (e instanceof ConsumerBusyException) {
+            }).exceptionally(e -> {
+                if (e.getCause() instanceof ConsumerBusyException) {
                     log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId,
                             consumerName);
-                } else if (e instanceof SubscriptionBusyException) {
+                } else if (e.getCause() instanceof SubscriptionBusyException) {
                     log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
                 }
 
                 decrementUsageCount();
                 future.completeExceptionally(e);
-            }
+                return null;
+            });
         }).exceptionally(ex -> {
             log.error("[{}] Failed to create subscription: {} error: {}", topic, subscriptionName, ex);
             decrementUsageCount();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java
index 2917ab9..abf71e9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java
@@ -98,7 +98,7 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen
                 .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount();
         this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar()
                 .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis();
-        this.topic.getBrokerService().getPulsar().getTransactionExecutor()
+        this.topic.getBrokerService().getPulsar().getTransactionReplayExecutor()
                 .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() {
                     @Override
                     public void recoverComplete() {
@@ -178,7 +178,7 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen
     }
 
     private void handleTransactionMessage(TxnID txnId, Position position) {
-        if (!ongoingTxns.containsKey(txnId)) {
+        if (!ongoingTxns.containsKey(txnId) && !aborts.containsKey(txnId)) {
             ongoingTxns.put(txnId, (PositionImpl) position);
             PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey());
             //max read position is less than first ongoing transaction message position, so entryId -1
@@ -191,6 +191,7 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen
     public CompletableFuture<TransactionBufferReader> openTransactionBufferReader(TxnID txnID, long startSequenceId) {
         return null;
     }
+
     @Override
     public CompletableFuture<Void> commitTxn(TxnID txnID, long lowWaterMark) {
         if (log.isDebugEnabled()) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java
index 4da82a2..0c34527 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java
@@ -148,9 +148,9 @@ public class TransactionBufferHandlerImpl implements TransactionBufferHandler, T
                 }
             });
         } catch (ExecutionException e) {
+            log.error("endTxn channel is not active exception", e);
             cache.invalidate(topic);
             cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage()));
-            pendingRequests.remove(requestId);
             op.recycle();
         }
         return cb;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java
index a7ab2ea..e4dd078 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java
@@ -53,8 +53,8 @@ public interface PendingAckHandle {
      * @throws NotAllowedException if Use this method incorrectly eg. not use
      * PositionImpl or cumulative ack with a list of positions.
      */
-    CompletableFuture<Void> individualAcknowledgeMessage(TxnID txnID,
-                                                         List<MutablePair<PositionImpl, Integer>> positions);
+    CompletableFuture<Void> individualAcknowledgeMessage(TxnID txnID, List<MutablePair<PositionImpl,
+            Integer>> positions);
 
     /**
      * Acknowledge message(s) for an ongoing transaction.
@@ -121,4 +121,18 @@ public interface PendingAckHandle {
      * @param position {@link Position} which position need to clear
      */
     void clearIndividualPosition(Position position);
-}
+
+    /**
+     * Pending ack recover whether ready future.
+     *
+     * @return the future of result.
+     */
+    CompletableFuture<PendingAckHandle> pendingAckHandleFuture();
+
+    /**
+     * Close the pending ack handle.
+     *
+     * @return the future of this operation.
+     */
+    CompletableFuture<Void> close();
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java
new file mode 100644
index 0000000..3f2cc51
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack;
+
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry;
+
+/**
+ * Call back for pending ack reply.
+ */
+public interface PendingAckReplyCallBack {
+
+    /**
+     * Pending ack replay complete callback for pending ack store.
+     */
+    void replayComplete();
+
+    /**
+     * Handle metadata entry.
+     *
+     * @param pendingAckMetadataEntry {@link PendingAckMetadataEntry} the metadata entry of pending ack
+     */
+    void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java
new file mode 100644
index 0000000..3da676e
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.common.api.proto.CommandAck.AckType;
+
+/**
+ * To store transaction pending ack.
+ */
+public interface PendingAckStore {
+    /**
+     * Replay pending ack to recover the pending ack subscription pending ack state.
+     *
+     * @param pendingAckHandle the handle of pending ack
+     * @param executorService the replay executor service
+     */
+    void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService);
+
+    /**
+     * Close the transaction pending ack store.
+     *
+     * @return a future represents the result of this operation
+     */
+    CompletableFuture<Void> closeAsync();
+
+    /**
+     * Append the individual pending ack operation to the ack persistent store.
+     *
+     * @param txnID {@link TxnID} transaction id.
+     * @param positions {@link List} the list of position and batch size.
+     * @return a future represents the result of this operation
+     */
+    CompletableFuture<Void> appendIndividualAck(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions);
+
+    /**
+     * Append the cumulative pending ack operation to the ack persistent store.
+     *
+     * @param txnID {@link TxnID} transaction id.
+     * @param position {@link PositionImpl} the pending ack position.
+     * @return a future represents the result of this operation
+     */
+    CompletableFuture<Void> appendCumulativeAck(TxnID txnID, PositionImpl position);
+
+    /**
+     * Append the pending ack commit mark to the ack persistent store.
+     *
+     * @param txnID {@link TxnID} the transaction id for add commit mark.
+     * @param ackType {@link AckType} the ack type of the commit.
+     * @return a future represents the result of this operation
+     */
+    CompletableFuture<Void> appendCommitMark(TxnID txnID, AckType ackType);
+
+    /**
+     * Append the pending ack abort mark to the ack persistent store.
+     *
+     * @param txnID {@link Position} the txnID
+     * @param ackType {@link AckType} the ack type of the abort.
+     * @return a future represents the result of this operation
+     */
+    CompletableFuture<Void> appendAbortMark(TxnID txnID, AckType ackType);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java
new file mode 100644
index 0000000..e1a14d7
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+
+/**
+ * Provider of transaction pending ack store.
+ */
+public interface TransactionPendingAckStoreProvider {
+
+    /**
+     * Construct a provider from the provided class.
+     *
+     * @param providerClassName {@link String} the provider class name
+     * @return an instance of transaction buffer provider.
+     */
+    static TransactionPendingAckStoreProvider newProvider(String providerClassName) throws IOException {
+        Class<?> providerClass;
+        try {
+            providerClass = Class.forName(providerClassName);
+            Object obj = providerClass.newInstance();
+            checkArgument(obj instanceof TransactionPendingAckStoreProvider,
+                    "The factory has to be an instance of "
+                            + TransactionPendingAckStoreProvider.class.getName());
+
+            return (TransactionPendingAckStoreProvider) obj;
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+    }
+
+    /**
+     * Open the pending ack store.
+     *
+     * @param subscription {@link PersistentSubscription}
+     * @return a future represents the result of the operation.
+     *         an instance of {@link PendingAckStore} is returned
+     *         if the operation succeeds.
+     */
+    CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription);
+
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java
new file mode 100644
index 0000000..0f41e2c
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.exceptions;
+
+import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionBufferException;
+
+/**
+ * Transaction pending ack store provider exception.
+ */
+public class TransactionPendingAckStoreProviderException extends TransactionBufferException {
+
+    public TransactionPendingAckStoreProviderException(String message) {
+        super(message);
+    }
+
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java
new file mode 100644
index 0000000..314ccee
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * The implementation for pending ack exceptions.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.exceptions;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java
new file mode 100644
index 0000000..d882c80
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.common.api.proto.CommandAck.AckType;
+
+/**
+ * In memory implementation of {@link PendingAckStore}.
+ */
+public class InMemoryPendingAckStore implements PendingAckStore {
+
+    @Override
+    public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService scheduledExecutorService) {
+        pendingAckHandle.changeToReadyState();
+    }
+
+    @Override
+    public CompletableFuture<Void> closeAsync() {
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendIndividualAck(TxnID txnID,
+                                                       List<MutablePair<PositionImpl, Integer>> positions) {
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendCumulativeAck(TxnID txnID, PositionImpl position) {
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendCommitMark(TxnID txnID, AckType ackType) {
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendAbortMark(TxnID txnID, AckType ackType) {
+        return CompletableFuture.completedFuture(null);
+    }
+
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java
new file mode 100644
index 0000000..1958236
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
+
+public class InMemoryPendingAckStoreProvider implements TransactionPendingAckStoreProvider {
+
+    @Override
+    public CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription) {
+        return CompletableFuture.completedFuture(new InMemoryPendingAckStore());
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java
new file mode 100644
index 0000000..dd7989d
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack;
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata;
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.common.api.proto.CommandAck.AckType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MLPendingAckStore reply call back.
+ */
+public class MLPendingAckReplyCallBack implements PendingAckReplyCallBack {
+
+    private final PendingAckHandleImpl pendingAckHandle;
+
+    public MLPendingAckReplyCallBack(PendingAckHandleImpl pendingAckHandle) {
+        this.pendingAckHandle = pendingAckHandle;
+    }
+
+    @Override
+    public void replayComplete() {
+        log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!",
+                pendingAckHandle.getTopicName(), pendingAckHandle.getSubName());
+
+        if (pendingAckHandle.changeToReadyState()) {
+            pendingAckHandle.completeHandleFuture();
+            log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!",
+                    pendingAckHandle.getTopicName(), pendingAckHandle.getSubName());
+        } else {
+            log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!",
+                    pendingAckHandle.getTopicName(), pendingAckHandle.getSubName());
+        }
+    }
+
+    @Override
+    public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) {
+        TxnID txnID = new TxnID(pendingAckMetadataEntry.getTxnidMostBits(),
+                pendingAckMetadataEntry.getTxnidLeastBits());
+        AckType ackType = pendingAckMetadataEntry.getAckType();
+        switch (pendingAckMetadataEntry.getPendingAckOp()) {
+            case ABORT:
+                pendingAckHandle.handleAbort(txnID, ackType);
+                break;
+            case COMMIT:
+                pendingAckHandle.handleCommit(txnID, ackType, Collections.emptyMap());
+                break;
+            case ACK:
+                if (ackType == AckType.Cumulative) {
+                    PendingAckMetadata pendingAckMetadata =
+                            pendingAckMetadataEntry.getPendingAckMetadatasList().get(0);
+                    pendingAckHandle.handleCumulativeAckRecover(txnID,
+                            PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()));
+                } else {
+                    List<MutablePair<PositionImpl, Integer>> positions = new ArrayList<>();
+                    pendingAckMetadataEntry.getPendingAckMetadatasList().forEach(pendingAckMetadata -> {
+                        if (pendingAckMetadata.getAckSetsCount() == 0) {
+                            positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(),
+                                    pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize()));
+                        } else {
+                            PositionImpl position =
+                                    PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId());
+                            if (pendingAckMetadata.getAckSetsCount() > 0) {
+                                long[] ackSets = new long[pendingAckMetadata.getAckSetsCount()];
+                                for (int i = 0; i < pendingAckMetadata.getAckSetsCount(); i++) {
+                                    ackSets[i] = pendingAckMetadata.getAckSetAt(i);
+                                }
+                                position.setAckSet(ackSets);
+                            }
+                            positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize()));
+                        }
+                    });
+                    pendingAckHandle.handleIndividualAckRecover(txnID, positions);
+                }
+                break;
+            default:
+                throw new IllegalStateException("Transaction pending ack replay "
+                        + "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp());
+
+        }
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(MLPendingAckReplyCallBack.class);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java
new file mode 100644
index 0000000..14aaa5f
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import com.google.common.collect.ComparisonChain;
+import io.netty.buffer.ByteBuf;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata;
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry;
+import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckOp;
+import org.apache.pulsar.client.api.transaction.TxnID;
+import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
+import org.apache.pulsar.common.api.proto.CommandAck.AckType;
+import org.apache.pulsar.common.naming.TopicName;
+import org.jctools.queues.MessagePassingQueue;
+import org.jctools.queues.SpscArrayQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The implement of the pending ack store by manageLedger.
+ */
+public class MLPendingAckStore implements PendingAckStore {
+
+
+    private final ManagedLedger managedLedger;
+
+    private final ManagedCursor cursor;
+
+    public static final String PENDING_ACK_STORE_SUFFIX = "__transaction_pending_ack";
+
+    private static final String PENDING_ACK_STORE_CURSOR_NAME = "__pending_ack_state";
+
+    private final SpscArrayQueue<Entry> entryQueue;
+
+    //this is for replay
+    private final PositionImpl lastConfirmedEntry;
+
+    private PositionImpl currentLoadPosition;
+
+    /**
+     * The map is for pending ack store clear useless data.
+     * <p>
+     *     When ack message append to pending ack store, it will store the position which is persistent as key.
+     * <p>
+     *     When ack message append to pending ack store, it will store the position which is the max position of this
+     *     ack by the original topic as value.
+     * <p>
+     *     It will judge the position with the max sub cursor position whether smaller than the subCursor mark
+     *     delete position.
+     *     <p>
+     *         If the max position is smaller than the subCursor mark delete position, the log cursor will mark delete
+     *         the position.
+     */
+    private final ConcurrentSkipListMap<PositionImpl, PositionImpl> metadataPositions;
+
+    private final ManagedCursor subManagedCursor;
+
+    public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor,
+                             ManagedCursor subManagedCursor) {
+        this.managedLedger = managedLedger;
+        this.cursor = cursor;
+        this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition();
+        this.entryQueue = new SpscArrayQueue<>(2000);
+        this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry();
+        this.metadataPositions = new ConcurrentSkipListMap<>();
+        this.subManagedCursor = subManagedCursor;
+    }
+
+    @Override
+    public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService transactionReplayExecutor) {
+        transactionReplayExecutor
+                .execute(new PendingAckReplay(new MLPendingAckReplyCallBack(pendingAckHandle)));
+    }
+
+    //TODO can control the number of entry to read
+    private void readAsync(int numberOfEntriesToRead,
+                           AsyncCallbacks.ReadEntriesCallback readEntriesCallback) {
+        cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionImpl.latest);
+    }
+
+    @Override
+    public CompletableFuture<Void> closeAsync() {
+        CompletableFuture<Void> completableFuture = new CompletableFuture<>();
+        cursor.asyncClose(new AsyncCallbacks.CloseCallback() {
+            @Override
+            public void closeComplete(Object ctx) {
+                try {
+                    managedLedger.close();
+                } catch (Exception e) {
+                    completableFuture.completeExceptionally(e);
+                }
+                completableFuture.complete(null);
+            }
+
+            @Override
+            public void closeFailed(ManagedLedgerException exception, Object ctx) {
+                completableFuture.completeExceptionally(exception);
+            }
+        }, null);
+        return completableFuture;
+    }
+
+    @Override
+    public CompletableFuture<Void> appendIndividualAck(TxnID txnID,
+                                                       List<MutablePair<PositionImpl, Integer>> positions) {
+        PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry();
+        pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK);
+        pendingAckMetadataEntry.setAckType(AckType.Individual);
+        List<PendingAckMetadata> pendingAckMetadataList = new ArrayList<>();
+        positions.forEach(positionIntegerMutablePair -> {
+            PendingAckMetadata pendingAckMetadata = new PendingAckMetadata();
+            PositionImpl position = positionIntegerMutablePair.getLeft();
+            int batchSize = positionIntegerMutablePair.getRight();
+            if (positionIntegerMutablePair.getLeft().getAckSet() != null) {
+                for (long l : position.getAckSet()) {
+                    pendingAckMetadata.addAckSet(l);
+                }
+            }
+            pendingAckMetadata.setLedgerId(position.getLedgerId());
+            pendingAckMetadata.setEntryId(position.getEntryId());
+            pendingAckMetadata.setBatchSize(batchSize);
+            pendingAckMetadataList.add(pendingAckMetadata);
+        });
+        pendingAckMetadataEntry.addAllPendingAckMetadatas(pendingAckMetadataList);
+        return appendCommon(pendingAckMetadataEntry, txnID);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendCumulativeAck(TxnID txnID, PositionImpl position) {
+        PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry();
+        pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK);
+        pendingAckMetadataEntry.setAckType(AckType.Cumulative);
+        PendingAckMetadata pendingAckMetadata = new PendingAckMetadata();
+        if (position.getAckSet() != null) {
+            for (long l : position.getAckSet()) {
+                pendingAckMetadata.addAckSet(l);
+            }
+        }
+        pendingAckMetadata.setLedgerId(position.getLedgerId());
+        pendingAckMetadata.setEntryId(position.getEntryId());
+        pendingAckMetadataEntry.addAllPendingAckMetadatas(Collections.singleton(pendingAckMetadata));
+        return appendCommon(pendingAckMetadataEntry, txnID);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendCommitMark(TxnID txnID, AckType ackType) {
+        PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry();
+        pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.COMMIT);
+        pendingAckMetadataEntry.setAckType(ackType);
+        return appendCommon(pendingAckMetadataEntry, txnID);
+    }
+
+    @Override
+    public CompletableFuture<Void> appendAbortMark(TxnID txnID, AckType ackType) {
+        PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry();
+        pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ABORT);
+        pendingAckMetadataEntry.setAckType(ackType);
+        return appendCommon(pendingAckMetadataEntry, txnID);
+    }
+
+    private CompletableFuture<Void> appendCommon(PendingAckMetadataEntry pendingAckMetadataEntry, TxnID txnID) {
+        CompletableFuture<Void> completableFuture = new CompletableFuture<>();
+        pendingAckMetadataEntry.setTxnidLeastBits(txnID.getLeastSigBits());
+        pendingAckMetadataEntry.setTxnidMostBits(txnID.getMostSigBits());
+        int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize();
+        ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(transactionMetadataEntrySize, transactionMetadataEntrySize);
+        pendingAckMetadataEntry.writeTo(buf);
+        managedLedger.asyncAddEntry(buf, new AsyncCallbacks.AddEntryCallback() {
+
+            @Override
+            public void addComplete(Position position, ByteBuf entryData, Object ctx) {
+                if (log.isDebugEnabled()) {
+                    log.debug("[{}][{}] MLPendingAckStore message append success at {} txnId: {}, operation : {}",
+                            managedLedger.getName(), ctx, position, txnID, pendingAckMetadataEntry.getPendingAckOp());
+                }
+                // store the persistent position in to memory
+                if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT
+                        && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) {
+                    Optional<PendingAckMetadata> optional = pendingAckMetadataEntry.getPendingAckMetadatasList()
+                            .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(),
+                                    o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result());
+                    optional.ifPresent(pendingAckMetadata ->
+                            metadataPositions.compute((PositionImpl) position, (thisPosition, otherPosition) -> {
+                                PositionImpl nowPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(),
+                                        pendingAckMetadata.getEntryId());
+                                if (otherPosition == null) {
+                                    return nowPosition;
+                                } else {
+                                    return nowPosition.compareTo(otherPosition) > 0 ? nowPosition : otherPosition;
+                                }
+                    }));
+                }
+
+                buf.release();
+                completableFuture.complete(null);
+
+                if (!metadataPositions.isEmpty()) {
+                    PositionImpl firstPosition = metadataPositions.firstEntry().getKey();
+                    PositionImpl deletePosition = metadataPositions.firstEntry().getKey();
+                    while (!metadataPositions.isEmpty()
+                            && metadataPositions.firstKey() != null
+                            && subManagedCursor.getPersistentMarkDeletedPosition() != null
+                            && metadataPositions.firstEntry().getValue()
+                            .compareTo((PositionImpl) subManagedCursor.getPersistentMarkDeletedPosition()) <= 0) {
+                        deletePosition = metadataPositions.firstKey();
+                        metadataPositions.remove(metadataPositions.firstKey());
+                    }
+
+                    if (firstPosition != deletePosition) {
+                        PositionImpl finalDeletePosition = deletePosition;
+                        cursor.asyncMarkDelete(deletePosition,
+                                new AsyncCallbacks.MarkDeleteCallback() {
+                                    @Override
+                                    public void markDeleteComplete(Object ctx) {
+                                        if (log.isDebugEnabled()) {
+                                            log.debug("[{}] Transaction pending ack store mark delete position : "
+                                                            + "[{}] success", managedLedger.getName(),
+                                                    finalDeletePosition);
+                                        }
+                                    }
+
+                                    @Override
+                                    public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
+                                        if (log.isDebugEnabled()) {
+                                            log.error("[{}] Transaction pending ack store mark delete position : "
+                                                            + "[{}] fail!", managedLedger.getName(),
+                                                    finalDeletePosition, exception);
+                                        }
+                                    }
+                                }, null);
+                    }
+                }
+            }
+
+            @Override
+            public void addFailed(ManagedLedgerException exception, Object ctx) {
+                log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}",
+                        managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp());
+                buf.release();
+                completableFuture.completeExceptionally(new PersistenceException(exception));
+            }
+        } , null);
+        return completableFuture;
+    }
+
+    class PendingAckReplay implements Runnable {
+
+        private final FillEntryQueueCallback fillEntryQueueCallback;
+        private final PendingAckReplyCallBack pendingAckReplyCallBack;
+
+        PendingAckReplay(PendingAckReplyCallBack pendingAckReplyCallBack) {
+            this.fillEntryQueueCallback = new FillEntryQueueCallback();
+            this.pendingAckReplyCallBack = pendingAckReplyCallBack;
+        }
+
+        @Override
+        public void run() {
+            try {
+                while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) {
+                    if (((ManagedCursorImpl) cursor).isClosed()) {
+                        log.warn("[{}] MLPendingAckStore cursor have been closed, close replay thread.",
+                                cursor.getManagedLedger().getName());
+                        return;
+                    }
+                    fillEntryQueueCallback.fillQueue();
+                    Entry entry = entryQueue.poll();
+                    if (entry != null) {
+                        ByteBuf buffer = entry.getDataBuffer();
+                        currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId());
+                        PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry();
+                        pendingAckMetadataEntry.parseFrom(buffer, buffer.readableBytes());
+                        // store the persistent position in to memory
+                        // store the max position of this entry retain
+                        if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT
+                                && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) {
+                            Optional<PendingAckMetadata> optional = pendingAckMetadataEntry.getPendingAckMetadatasList()
+                                    .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(),
+                                            o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result());
+
+                            optional.ifPresent(pendingAckMetadata ->
+                                    metadataPositions.compute(PositionImpl.get(entry.getLedgerId(), entry.getEntryId()),
+                                            (thisPosition, otherPosition) -> {
+                                                PositionImpl nowPosition = PositionImpl
+                                                        .get(pendingAckMetadata.getLedgerId(),
+                                                                pendingAckMetadata.getEntryId());
+                                                if (otherPosition == null) {
+                                                    return nowPosition;
+                                                } else {
+                                                    return nowPosition.compareTo(otherPosition) > 0 ? nowPosition
+                                                            : otherPosition;
+                                                }
+                                            }));
+                        }
+                        pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry);
+                        entry.release();
+                    } else {
+                        try {
+                            Thread.sleep(1);
+                        } catch (InterruptedException e) {
+                            if (Thread.interrupted()) {
+                                log.error("[{}]Transaction pending "
+                                        + "replay thread interrupt!", managedLedger.getName(), e);
+                            }
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                log.error("[{}] Pending ack recover fail!", subManagedCursor.getManagedLedger().getName(), e);
+                return;
+            }
+            pendingAckReplyCallBack.replayComplete();
+        }
+    }
+
+    class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback {
+
+        private final AtomicLong outstandingReadsRequests = new AtomicLong(0);
+
+        void fillQueue() {
+            if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) {
+                if (cursor.hasMoreEntries()) {
+                    outstandingReadsRequests.incrementAndGet();
+                    readAsync(100, this);
+                }
+            }
+        }
+
+        @Override
+        public void readEntriesComplete(List<Entry> entries, Object ctx) {
+            entryQueue.fill(new MessagePassingQueue.Supplier<Entry>() {
+                private int i = 0;
+                @Override
+                public Entry get() {
+                    Entry entry = entries.get(i);
+                    i++;
+                    return entry;
+                }
+            }, entries.size());
+
+            outstandingReadsRequests.decrementAndGet();
+        }
+
+        @Override
+        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+            log.error("MLPendingAckStore stat reply fail!", exception);
+            outstandingReadsRequests.decrementAndGet();
+        }
+
+    }
+
+    public static String getTransactionPendingAckStoreSuffix(String originTopicName, String subName) {
+        return TopicName.get(originTopicName) + "-" + subName + PENDING_ACK_STORE_SUFFIX;
+    }
+
+    public static String getTransactionPendingAckStoreCursorName() {
+        return PENDING_ACK_STORE_CURSOR_NAME;
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(MLPendingAckStore.class);
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java
new file mode 100644
index 0000000..0741f06
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import java.util.concurrent.CompletableFuture;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.AsyncCallbacks;
+import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
+import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException;
+import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
+import org.apache.pulsar.common.naming.TopicName;
+
+/**
+ * Provider is for MLPendingAckStore.
+ */
+@Slf4j
+public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider {
+
+    @Override
+    public CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription) {
+        CompletableFuture<PendingAckStore> pendingAckStoreFuture = new CompletableFuture<>();
+
+        if (subscription == null) {
+            pendingAckStoreFuture.completeExceptionally(
+                    new TransactionPendingAckStoreProviderException("The subscription is null."));
+            return pendingAckStoreFuture;
+        }
+
+        PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic();
+        String pendingAckTopicName = MLPendingAckStore
+                .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subscription.getName());
+
+        originPersistentTopic.getBrokerService().getManagedLedgerFactory()
+                .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(),
+                        originPersistentTopic.getManagedLedger().getConfig(),
+                        new AsyncCallbacks.OpenLedgerCallback() {
+                            @Override
+                            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
+                                ledger.asyncOpenCursor(MLPendingAckStore.getTransactionPendingAckStoreCursorName(),
+                                        InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() {
+                                            @Override
+                                            public void openCursorComplete(ManagedCursor cursor, Object ctx) {
+                                                pendingAckStoreFuture
+                                                        .complete(new MLPendingAckStore(ledger, cursor,
+                                                                subscription.getCursor()));
+                                            }
+
+                                            @Override
+                                            public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
+                                                log.error("Open MLPendingAckStore cursor failed.", exception);
+                                                pendingAckStoreFuture.completeExceptionally(exception);
+                                            }
+                                        }, null);
+                            }
+
+                            @Override
+                            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
+                                log.error("Open MLPendingAckStore managedLedger failed.", exception);
+                                pendingAckStoreFuture.completeExceptionally(exception);
+                            }
+                        }, () -> true, null);
+        return pendingAckStoreFuture;
+    }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java
index f58054a..cb86ff7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java
@@ -35,6 +35,9 @@ import org.apache.pulsar.common.util.FutureUtil;
  */
 public class PendingAckHandleDisabled implements PendingAckHandle {
 
+    private final CompletableFuture<PendingAckHandle> pendingAckHandleCompletableFuture =
+            CompletableFuture.completedFuture(PendingAckHandleDisabled.this);
+
     @Override
     public CompletableFuture<Void> individualAcknowledgeMessage(TxnID txnID,
                                                                 List<MutablePair<PositionImpl, Integer>> positions) {
@@ -68,6 +71,17 @@ public class PendingAckHandleDisabled implements PendingAckHandle {
 
     @Override
     public void clearIndividualPosition(Position position) {
-        //no operation
+        //no-op
+    }
+
+    @Override
+    public CompletableFuture<PendingAckHandle> pendingAckHandleFuture() {
+        return pendingAckHandleCompletableFuture;
     }
+
+    @Override
+    public CompletableFuture<Void> close() {
+        return CompletableFuture.completedFuture(null);
+    }
+
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java
index d08b402..dcc4377 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java
@@ -26,7 +26,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentSkipListMap;
 import lombok.extern.slf4j.Slf4j;
@@ -37,9 +36,13 @@ import org.apache.commons.collections4.map.LinkedMap;
 import org.apache.commons.lang3.tuple.MutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException;
+import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
 import org.apache.pulsar.client.api.transaction.TxnID;
 import org.apache.pulsar.common.api.proto.CommandAck.AckType;
 import org.apache.pulsar.common.util.FutureUtil;
@@ -50,7 +53,7 @@ import org.apache.pulsar.transaction.common.exception.TransactionConflictExcepti
  * The default implementation of {@link PendingAckHandle}.
  */
 @Slf4j
-public class PendingAckHandleImpl implements PendingAckHandle {
+public class PendingAckHandleImpl extends PendingAckHandleState implements PendingAckHandle {
 
     /**
      * The map is for transaction with position witch was individual acked by this transaction.
@@ -95,14 +98,35 @@ public class PendingAckHandleImpl implements PendingAckHandle {
 
     private final PersistentSubscription persistentSubscription;
 
+    private final CompletableFuture<PendingAckStore> pendingAckStoreFuture;
+
+    private final CompletableFuture<PendingAckHandle> pendingAckHandleCompletableFuture = new CompletableFuture<>();
+
     public PendingAckHandleImpl(PersistentSubscription persistentSubscription) {
+        super(State.None);
         this.topicName = persistentSubscription.getTopicName();
         this.subName = persistentSubscription.getName();
         this.persistentSubscription = persistentSubscription;
+
+        TransactionPendingAckStoreProvider pendingAckStoreProvider =
+                ((PersistentTopic) this.persistentSubscription.getTopic())
+                        .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider();
+        this.pendingAckStoreFuture =
+                pendingAckStoreProvider.newPendingAckStore(persistentSubscription);
+
+        this.pendingAckStoreFuture.thenAccept(pendingAckStore -> {
+            changeToInitializingState();
+            pendingAckStore.replayAsync(this,
+                    ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService()
+                            .getPulsar().getTransactionReplayExecutor());
+        }).exceptionally(e -> {
+            log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e);
+            return null;
+        });
     }
 
     @Override
-    public synchronized CompletableFuture<Void> individualAcknowledgeMessage(TxnID txnID,
+    public CompletableFuture<Void> individualAcknowledgeMessage(TxnID txnID,
                                                                 List<MutablePair<PositionImpl, Integer>> positions) {
         if (txnID == null) {
             return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null."));
@@ -111,99 +135,104 @@ public class PendingAckHandleImpl implements PendingAckHandle {
             return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null."));
         }
         CompletableFuture<Void> completableFuture = new CompletableFuture<>();
-        for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
-            PositionImpl position = positionIntegerMutablePair.left;
-
-            // If try to ack message already acked by committed transaction or normal acknowledge, throw exception.
-            if (((ManagedCursorImpl) persistentSubscription.getCursor())
-                    .isMessageDeleted(position)) {
-                String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                        + " try to ack message:" + position + " already acked before.";
-                log.error(errorMsg);
-                return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-            }
-
-            if (position.hasAckSet()) {
-                //in order to jude the bit set is over lap, so set the covering the batch size bit to 1,
-                // should know the two bit set don't have the same point is 0
-                BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet());
-                if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
-                    bitSetRecyclable.set(positionIntegerMutablePair.right);
-                }
-                bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
-                long[] ackSetOverlap = bitSetRecyclable.toLongArray();
-                bitSetRecyclable.recycle();
-                if (isAckSetOverlap(ackSetOverlap,
-                        ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) {
-                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                            + " try to ack message:" + position + " already acked before.";
-                    log.error(errorMsg);
-                    return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-                }
 
-                if (this.individualAckPositions != null && individualAckPositions.containsKey(position)
-                        && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) {
-                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                            + " try to ack batch message:" + position + " in pending ack status.";
-                    log.error(errorMsg);
-                    return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-                }
-            } else {
-                if (this.individualAckPositions != null && this.individualAckPositions.containsKey(position)) {
-                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                            + " try to ack message:" + position + " in pending ack status.";
-                    log.error(errorMsg);
-                    return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-                }
-            }
-        }
-        for (int i = 0; i < positions.size(); i++) {
-            if (log.isDebugEnabled()) {
-                log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, subName, txnID.toString(), positions);
-            }
-            if (individualAckOfTransaction == null) {
-                individualAckOfTransaction = new LinkedMap<>();
-            }
-
-            if (individualAckPositions == null) {
-                individualAckPositions = new ConcurrentSkipListMap<>();
-            }
-
-            PositionImpl position = positions.get(i).left;
-
-            if (position.hasAckSet()) {
-
-                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
-                        individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>());
-
-                if (pendingAckMessageForCurrentTxn.containsKey(position)) {
-                    andAckSet(pendingAckMessageForCurrentTxn.get(position), position);
-                } else {
-                    pendingAckMessageForCurrentTxn.put(position, position);
-                }
-
-                if (!individualAckPositions.containsKey(position)) {
-                    this.individualAckPositions.put(position, positions.get(i));
-                } else {
-                    MutablePair<PositionImpl, Integer> positionPair = this.individualAckPositions.get(position);
-                    positionPair.setRight(positions.get(i).right);
-                    andAckSet(positionPair.getLeft(), position);
-                }
-
-            } else {
-                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
-                        individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>());
-                pendingAckMessageForCurrentTxn.put(position, position);
-                this.individualAckPositions.putIfAbsent(position, positions.get(i));
-            }
-        }
-        completableFuture.complete(null);
+        this.pendingAckStoreFuture.thenAccept(pendingAckStore ->
+                pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> {
+                    synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) {
+                        for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
+
+                            if (log.isDebugEnabled()) {
+                                log.debug("[{}] individualAcknowledgeMessage position: [{}], "
+                                                + "txnId: [{}], subName: [{}]", topicName,
+                                        positionIntegerMutablePair.left, txnID, subName);
+                            }
+                            PositionImpl position = positionIntegerMutablePair.left;
+
+                            // If try to ack message already acked by committed transaction or
+                            // normal acknowledge,throw exception.
+                            if (((ManagedCursorImpl) persistentSubscription.getCursor())
+                                    .isMessageDeleted(position)) {
+                                String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
+                                        + " try to ack message:" + position + " already acked before.";
+                                log.error(errorMsg);
+                                completableFuture
+                                        .completeExceptionally(new TransactionConflictException(errorMsg));
+                                return;
+                            }
+
+                            if (position.hasAckSet()) {
+                                //in order to jude the bit set is over lap, so set the covering
+                                // the batch size bit to 1,should know the two
+                                // bit set don't have the same point is 0
+                                BitSetRecyclable bitSetRecyclable =
+                                        BitSetRecyclable.valueOf(position.getAckSet());
+                                if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
+                                    bitSetRecyclable.set(positionIntegerMutablePair.right);
+                                }
+                                bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
+                                long[] ackSetOverlap = bitSetRecyclable.toLongArray();
+                                bitSetRecyclable.recycle();
+                                if (isAckSetOverlap(ackSetOverlap,
+                                        ((ManagedCursorImpl) persistentSubscription.getCursor())
+                                                .getBatchPositionAckSet(position))) {
+                                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:"
+                                            + txnID + " try to ack message:"
+                                            + position + " already acked before.";
+                                    log.error(errorMsg);
+                                    completableFuture
+                                            .completeExceptionally(new TransactionConflictException(errorMsg));
+                                    return;
+                                }
+
+                                if (individualAckPositions != null
+                                        && individualAckPositions.containsKey(position)
+                                        && isAckSetOverlap(individualAckPositions
+                                        .get(position).getLeft().getAckSet(), ackSetOverlap)) {
+                                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:"
+                                            + txnID + " try to ack batch message:"
+                                            + position + " in pending ack status.";
+                                    log.error(errorMsg);
+                                    completableFuture
+                                            .completeExceptionally(new TransactionConflictException(errorMsg));
+                                    return;
+                                }
+                            } else {
+                                if (individualAckPositions != null
+                                        && individualAckPositions.containsKey(position)) {
+                                    String errorMsg = "[" + topicName + "][" + subName + "] Transaction:"
+                                            + txnID + " try to ack message:"
+                                            + position + " in pending ack status.";
+                                    log.error(errorMsg);
+                                    completableFuture
+                                            .completeExceptionally(new TransactionConflictException(errorMsg));
+                                    return;
+                                }
+                            }
+                        }
+
+                        handleIndividualAck(txnID, positions);
+                        completableFuture.complete(null);
+                    }
+                }).exceptionally(e -> {
+                    synchronized (PendingAckHandleImpl.this) {
+                        // we also modify the in memory state when append fail,
+                        // because we don't know the persistent state, when were replay it,
+                        // it will produce the wrong operation. so we append fail,
+                        // we should wait tc time out or client abort this transaction.
+                        handleIndividualAck(txnID, positions);
+                        completableFuture.completeExceptionally(e.getCause());
+                    }
+                    return null;
+                })).exceptionally(e -> {
+            completableFuture.completeExceptionally(e);
+            return null;
+        });
         return completableFuture;
     }
 
     @Override
-    public synchronized CompletableFuture<Void> cumulativeAcknowledgeMessage(TxnID txnID,
-                                                                             List<PositionImpl> positions) {
+    public CompletableFuture<Void> cumulativeAcknowledgeMessage(TxnID txnID,
+                                                                List<PositionImpl> positions) {
 
         if (txnID == null) {
             return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null."));
@@ -221,94 +250,182 @@ public class PendingAckHandleImpl implements PendingAckHandle {
 
         PositionImpl position = positions.get(0);
 
-        if (position.compareTo((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) <= 0) {
-            String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                    + " try to cumulative ack position: " + position + " within range of cursor's "
-                    + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition();
-            log.error(errorMsg);
-            return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-        }
+        CompletableFuture<Void> completableFuture = new CompletableFuture<>();
 
-        if (log.isDebugEnabled()) {
-            log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position);
-        }
+        this.pendingAckStoreFuture.thenAccept(pendingAckStore ->
+                pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> {
+                    if (log.isDebugEnabled()) {
+                        log.debug("[{}] cumulativeAcknowledgeMessage position: [{}], "
+                                + "txnID:[{}], subName: [{}].", topicName, txnID.toString(), position, subName);
+                    }
 
-        if (this.cumulativeAckOfTransaction == null) {
-            this.cumulativeAckOfTransaction = MutablePair.of(txnID, position);
-        } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID)
-                && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) {
-            this.cumulativeAckOfTransaction.setValue(position);
+                    if (position.compareTo((PositionImpl) persistentSubscription.getCursor()
+                            .getMarkDeletedPosition()) <= 0) {
+                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
+                                + " try to cumulative ack position: " + position + " within range of cursor's "
+                                + "markDeletePosition: "
+                                + persistentSubscription.getCursor().getMarkDeletedPosition();
+                        log.error(errorMsg);
+                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
+                        return;
+                    }
 
-        } else {
-            String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
-                    + " try to cumulative batch ack position: " + position + " within range of current "
-                    + "currentPosition: " + this.cumulativeAckOfTransaction.getValue();
-            log.error(errorMsg);
-            return FutureUtil.failedFuture(new TransactionConflictException(errorMsg));
-        }
-        return CompletableFuture.completedFuture(null);
+                    if (cumulativeAckOfTransaction != null && (!cumulativeAckOfTransaction.getKey().equals(txnID)
+                            || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0)) {
+                        String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID
+                                + " try to cumulative batch ack position: " + position + " within range of current "
+                                + "currentPosition: " + cumulativeAckOfTransaction.getValue();
+                        log.error(errorMsg);
+                        completableFuture.completeExceptionally(new TransactionConflictException(errorMsg));
+                        return;
+                    }
+
+                    handleCumulativeAck(txnID, position);
+                    completableFuture.complete(null);
+                }).exceptionally(e -> {
+                    //we also modify the in memory state when append fail, because we don't know the persistent
+                    // state, when wereplay it, it will produce the wrong operation. so we append fail, we should
+                    // wait tc time out or client abort this transaction.
+                    handleCumulativeAck(txnID, position);
+                    completableFuture.completeExceptionally(e.getCause());
+                    return null;
+                })
+        ).exceptionally(e -> {
+            completableFuture.completeExceptionally(e);
+            return null;
+        });
+        return completableFuture;
     }
 
     @Override
     public synchronized CompletableFuture<Void> commitTxn(TxnID txnID, Map<String, Long> properties,
                                                           long lowWaterMark) {
+        if (!checkIfReady()) {
+            return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!"));
+        }
 
         CompletableFuture<Void> commitFuture = new CompletableFuture<>();
+
         // It's valid to create transaction then commit without doing any operation, which will cause
         // pendingAckMessagesMap to be null.
         if (this.cumulativeAckOfTransaction != null) {
             if (cumulativeAckOfTransaction.getKey().equals(txnID)) {
-                persistentSubscription.acknowledgeMessage(Collections
-                        .singletonList(this.cumulativeAckOfTransaction.getValue()), AckType.Cumulative, properties);
-                this.cumulativeAckOfTransaction = null;
+                pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore
+                        .appendCommitMark(txnID, AckType.Cumulative).thenAccept(v -> {
+                            if (log.isDebugEnabled()) {
+                                log.debug("[{}] Transaction pending ack store commit txnId : [{}] "
+                                        + "success! subName: [{}]", topicName, txnID, subName);
+                            }
+                            persistentSubscription.acknowledgeMessage(
+                                    Collections.singletonList(cumulativeAckOfTransaction.getValue()),
+                                    AckType.Cumulative, properties);
+                            cumulativeAckOfTransaction = null;
+                            commitFuture.complete(null);
+                        }).exceptionally(e -> {
+                            log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!",
+                                    topicName, txnID, e);
+                            commitFuture.completeExceptionally(e);
+                            return null;
+                        })).exceptionally(e -> {
+                    commitFuture.completeExceptionally(e);
+                    return null;
+                });
+            } else {
+                commitFuture.complete(null);
             }
         } else {
-            if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) {
-                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
-                        individualAckOfTransaction.remove(txnID);
-                if (pendingAckMessageForCurrentTxn != null) {
-                    persistentSubscription.acknowledgeMessage(new ArrayList<>(pendingAckMessageForCurrentTxn.values()),
-                            AckType.Individual, properties);
-                }
-            }
+            pendingAckStoreFuture.thenAccept(pendingAckStore ->
+                    pendingAckStore.appendCommitMark(txnID, AckType.Individual).thenAccept(v -> {
+                        synchronized (PendingAckHandleImpl.this) {
+                            if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) {
+                                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                                        individualAckOfTransaction.get(txnID);
+                                if (log.isDebugEnabled()) {
+                                    log.debug("[{}] Transaction pending ack store commit txnId : "
+                                            + "[{}] success! subName: [{}]", topicName, txnID, subName);
+                                }
+                                individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties);
+                                commitFuture.complete(null);
+                                handleLowWaterMark(txnID, lowWaterMark);
+                            } else {
+                                commitFuture.complete(null);
+                            }
+                        }
+                    }).exceptionally(e -> {
+                        log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!",
+                                topicName, txnID, e);
+                        commitFuture.completeExceptionally(e.getCause());
+                        return null;
+                    })).exceptionally(e -> {
+                commitFuture.completeExceptionally(e);
+                return null;
+            });
         }
-        handleLowWaterMark(txnID, lowWaterMark);
-        commitFuture.complete(null);
         return commitFuture;
     }
 
     @Override
     public synchronized CompletableFuture<Void> abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark) {
+        if (!checkIfReady()) {
+            return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!"));
+        }
         CompletableFuture<Void> abortFuture = new CompletableFuture<>();
         if (this.cumulativeAckOfTransaction != null) {
-            if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) {
-                this.cumulativeAckOfTransaction = null;
-            }
-            this.persistentSubscription.redeliverUnacknowledgedMessages(consumer);
-        } else if (this.individualAckOfTransaction != null){
-            HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
-                    individualAckOfTransaction.remove(txnId);
-            if (pendingAckMessageForCurrentTxn != null) {
-                for (Entry<PositionImpl, PositionImpl> entry : pendingAckMessageForCurrentTxn.entrySet()) {
-                    if (entry.getValue().hasAckSet() && individualAckPositions.containsKey(entry.getValue())) {
-                        BitSetRecyclable thisBitSet = BitSetRecyclable.valueOf(entry.getValue().getAckSet());
-                        thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right);
-                        BitSetRecyclable otherBitSet =
-                                BitSetRecyclable.valueOf(individualAckPositions.get(entry.getValue()).left.getAckSet());
-                        otherBitSet.or(thisBitSet);
-                        individualAckPositions.get(entry.getKey()).left.setAckSet(otherBitSet.toLongArray());
-                        otherBitSet.recycle();
-                        thisBitSet.recycle();
-                    } else {
-                        individualAckPositions.remove(entry.getValue());
-                    }
-                }
-                this.persistentSubscription.redeliverUnacknowledgedMessages(consumer,
-                        new ArrayList<>(pendingAckMessageForCurrentTxn.values()));
-            }
+            pendingAckStoreFuture.thenAccept(pendingAckStore ->
+                    pendingAckStore.appendAbortMark(txnId, AckType.Cumulative).thenAccept(v -> {
+                        if (log.isDebugEnabled()) {
+                            log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! subName: [{}]",
+                                    topicName, txnId, subName);
+                        }
+                        if (cumulativeAckOfTransaction.getKey().equals(txnId)) {
+                            cumulativeAckOfTransaction = null;
+                        }
+                        persistentSubscription.redeliverUnacknowledgedMessages(consumer);
+                        abortFuture.complete(null);
+                    }).exceptionally(e -> {
+                        log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!",
+                                topicName, txnId, e);
+                        abortFuture.completeExceptionally(e);
+                        return null;
+                    })
+            ).exceptionally(e -> {
+                abortFuture.completeExceptionally(e);
+                return null;
+            });
+        } else if (this.individualAckOfTransaction != null) {
+            pendingAckStoreFuture.thenAccept(pendingAckStore ->
+                    pendingAckStore.appendAbortMark(txnId, AckType.Individual).thenAccept(v -> {
+                        synchronized (PendingAckHandleImpl.this) {
+                            HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                                    individualAckOfTransaction.get(txnId);
+                            if (pendingAckMessageForCurrentTxn != null) {
+                                if (log.isDebugEnabled()) {
+                                    log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! "
+                                            + "subName: [{}]", topicName, txnId, subName);
+                                }
+                                individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn);
+                                persistentSubscription.redeliverUnacknowledgedMessages(consumer,
+                                        new ArrayList<>(pendingAckMessageForCurrentTxn.values()));
+                                abortFuture.complete(null);
+                                handleLowWaterMark(txnId, lowWaterMark);
+                            } else {
+                                abortFuture.complete(null);
+                            }
+                        }
+                    }).exceptionally(e -> {
+                        log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!",
+                                topicName, txnId, e);
+                        abortFuture.completeExceptionally(e);
+                        return null;
+                    })
+            ).exceptionally(e -> {
+                log.error("[{}] abortTxn", txnId, e);
+                abortFuture.completeExceptionally(e);
+                return null;
+            });
+        } else {
+            abortFuture.complete(null);
         }
-        handleLowWaterMark(txnId, lowWaterMark);
-        abortFuture.complete(null);
         return abortFuture;
     }
 
@@ -318,8 +435,22 @@ public class PendingAckHandleImpl implements PendingAckHandle {
 
             if (firstTxn.getMostSigBits() == txnID.getMostSigBits()
                     && firstTxn.getLeastSigBits() <= lowWaterMark) {
-                individualAckOfTransaction.remove(firstTxn);
-                handleLowWaterMark(txnID, lowWaterMark);
+                this.pendingAckStoreFuture.whenComplete((pendingAckStore, throwable) -> {
+                    if (throwable == null) {
+                        pendingAckStore.appendAbortMark(txnID, AckType.Individual).thenAccept(v -> {
+                            synchronized (PendingAckHandleImpl.this) {
+                                log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], "
+                                        + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark);
+                                individualAckOfTransaction.remove(firstTxn);
+                                handleLowWaterMark(txnID, lowWaterMark);
+                            }
+                        }).exceptionally(e -> {
+                            log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], "
+                                    + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark);
+                            return null;
+                        });
+                    }
+                });
             }
         }
     }
@@ -329,8 +460,8 @@ public class PendingAckHandleImpl implements PendingAckHandle {
         if (individualAckPositions == null) {
             individualAckPositions = new ConcurrentSkipListMap<>();
         }
-        //sync don't carry the batch size
-        //when one position is ack by transaction the batch size is for `and` operation.
+        // sync don't carry the batch size
+        // when one position is ack by transaction the batch size is for `and` operation.
         if (!individualAckPositions.containsKey(position)) {
             this.individualAckPositions.put(position, new MutablePair<>(position, 0));
         } else {
@@ -339,7 +470,7 @@ public class PendingAckHandleImpl implements PendingAckHandle {
     }
 
     @Override
-    public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) {
+    public synchronized boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) {
         if (!individualAckPositions.containsKey(position)) {
             return true;
         } else {
@@ -359,24 +490,217 @@ public class PendingAckHandleImpl implements PendingAckHandle {
         }
     }
 
+    protected void handleAbort(TxnID txnID, AckType ackType) {
+        if (ackType == AckType.Cumulative) {
+            this.cumulativeAckOfTransaction = null;
+        } else {
+            if (this.individualAckOfTransaction != null) {
+                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                        individualAckOfTransaction.get(txnID);
+                if (pendingAckMessageForCurrentTxn != null) {
+                    individualAckAbortCommon(txnID, pendingAckMessageForCurrentTxn);
+                }
+            }
+        }
+    }
+
+    private void individualAckAbortCommon(TxnID txnID, HashMap<PositionImpl, PositionImpl> currentTxn) {
+        for (Map.Entry<PositionImpl, PositionImpl> entry :
+                currentTxn.entrySet()) {
+            if (entry.getValue().hasAckSet()
+                    && individualAckPositions.containsKey(entry.getValue())) {
+                BitSetRecyclable thisBitSet =
+                        BitSetRecyclable.valueOf(entry.getValue().getAckSet());
+                thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right);
+                BitSetRecyclable otherBitSet =
+                        BitSetRecyclable.valueOf(individualAckPositions
+                                .get(entry.getValue()).left.getAckSet());
+                otherBitSet.or(thisBitSet);
+                individualAckPositions.get(entry.getKey())
+                        .left.setAckSet(otherBitSet.toLongArray());
+                otherBitSet.recycle();
+                thisBitSet.recycle();
+            } else {
+                individualAckPositions.remove(entry.getValue());
+            }
+        }
+        individualAckOfTransaction.remove(txnID);
+    }
+
+    protected void handleCommit(TxnID txnID, AckType ackType, Map<String, Long> properties) {
+        if (ackType == AckType.Cumulative) {
+            if (this.cumulativeAckOfTransaction != null) {
+                persistentSubscription.acknowledgeMessage(
+                        Collections.singletonList(this.cumulativeAckOfTransaction.getValue()),
+                        AckType.Cumulative, properties);
+            }
+            this.cumulativeAckOfTransaction = null;
+        } else {
+            if (this.individualAckOfTransaction != null) {
+                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                        individualAckOfTransaction.get(txnID);
+                if (pendingAckMessageForCurrentTxn != null) {
+                    individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, null);
+                }
+            }
+        }
+    }
+
+    private void individualAckCommitCommon(TxnID txnID,
+                                           HashMap<PositionImpl, PositionImpl> currentTxn,
+                                           Map<String, Long> properties) {
+        if (currentTxn != null) {
+            persistentSubscription.acknowledgeMessage(new ArrayList<>(currentTxn.values()),
+                    AckType.Individual, properties);
+            individualAckOfTransaction.remove(txnID);
+        }
+    }
+
+    private void handleIndividualAck(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions) {
+        for (int i = 0; i < positions.size(); i++) {
+            if (log.isDebugEnabled()) {
+                log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName,
+                        subName, txnID.toString(), positions);
+            }
+            if (individualAckOfTransaction == null) {
+                individualAckOfTransaction = new LinkedMap<>();
+            }
+
+            if (individualAckPositions == null) {
+                individualAckPositions = new ConcurrentSkipListMap<>();
+            }
+
+            PositionImpl position = positions.get(i).left;
+
+            if (position.hasAckSet()) {
+
+                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                        individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>());
+
+                if (pendingAckMessageForCurrentTxn.containsKey(position)) {
+                    andAckSet(pendingAckMessageForCurrentTxn.get(position), position);
+                } else {
+                    pendingAckMessageForCurrentTxn.put(position, position);
+                }
+
+                if (!individualAckPositions.containsKey(position)) {
+                    this.individualAckPositions.put(position, positions.get(i));
+                } else {
+                    MutablePair<PositionImpl, Integer> positionPair =
+                            this.individualAckPositions.get(position);
+                    positionPair.setRight(positions.get(i).right);
+                    andAckSet(positionPair.getLeft(), position);
+                }
+
+            } else {
+                HashMap<PositionImpl, PositionImpl> pendingAckMessageForCurrentTxn =
+                        individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>());
+                pendingAckMessageForCurrentTxn.put(position, position);
+                this.individualAckPositions.putIfAbsent(position, positions.get(i));
+            }
+        }
+    }
+
+    private void handleCumulativeAck(TxnID txnID, PositionImpl position) {
+        if (this.cumulativeAckOfTransaction == null) {
+            this.cumulativeAckOfTransaction = MutablePair.of(txnID, position);
+        } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID)
+                && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) {
+            this.cumulativeAckOfTransaction.setValue(position);
+        }
+    }
+
+    protected void handleCumulativeAckRecover(TxnID txnID, PositionImpl position) {
+        if ((position.compareTo((PositionImpl) persistentSubscription.getCursor()
+                .getMarkDeletedPosition()) > 0) && (cumulativeAckOfTransaction == null
+                || (cumulativeAckOfTransaction.getKey().equals(txnID)
+                && compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) > 0))) {
+            handleCumulativeAck(txnID, position);
+        }
+    }
+
+    protected void handleIndividualAckRecover(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions) {
+        for (MutablePair<PositionImpl, Integer> positionIntegerMutablePair : positions) {
+            PositionImpl position = positionIntegerMutablePair.left;
+
+            // If try to ack message already acked by committed transaction or
+            // normal acknowledge,throw exception.
+            if (((ManagedCursorImpl) persistentSubscription.getCursor())
+                    .isMessageDeleted(position)) {
+                return;
+            }
+
+            if (position.hasAckSet()) {
+                //in order to jude the bit set is over lap, so set the covering
+                // the batch size bit to 1,should know the two
+                // bit set don't have the same point is 0
+                BitSetRecyclable bitSetRecyclable =
+                        BitSetRecyclable.valueOf(position.getAckSet());
+                if (positionIntegerMutablePair.right > bitSetRecyclable.size()) {
+                    bitSetRecyclable.set(positionIntegerMutablePair.right);
+                }
+                bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size());
+                long[] ackSetOverlap = bitSetRecyclable.toLongArray();
+                bitSetRecyclable.recycle();
+                if (isAckSetOverlap(ackSetOverlap,
+                        ((ManagedCursorImpl) persistentSubscription.getCursor())
+                                .getBatchPositionAckSet(position))) {
+                    return;
+                }
+
+                if (individualAckPositions != null
+                        && individualAckPositions.containsKey(position)
+                        && isAckSetOverlap(individualAckPositions
+                        .get(position).getLeft().getAckSet(), ackSetOverlap)) {
+                    return;
+                }
+            } else {
+                if (individualAckPositions != null
+                        && individualAckPositions.containsKey(position)) {
+                    return;
+                }
+            }
+        }
+        handleIndividualAck(txnID, positions);
+    }
+
+    public String getTopicName() {
+        return topicName;
+    }
+
+    public String getSubName() {
+        return subName;
+    }
+
     @Override
-    public void clearIndividualPosition(Position position) {
+    public synchronized void clearIndividualPosition(Position position) {
         if (individualAckPositions == null) {
             return;
         }
 
         if (position instanceof PositionImpl) {
             individualAckPositions.remove(position);
-            for (PositionImpl individualAckPosition : individualAckPositions.keySet()) {
-                // individualAckPositions is currentSkipListMap, delete the position form individualAckPositions which
-                // is smaller than can delete position
-                if (individualAckPosition.compareTo((PositionImpl) position) <= 0) {
-                    individualAckPositions.remove(individualAckPosition);
-                } else {
-                    return;
-                }
-            }
         }
+
+        individualAckPositions.forEach((persistentPosition, positionIntegerMutablePair) -> {
+            if (persistentPosition.compareTo((PositionImpl) persistentSubscription
+                    .getCursor().getMarkDeletedPosition()) < 0) {
+                individualAckPositions.remove(persistentPosition);
+            }
+        });
     }
 
-}
+    @Override
+    public CompletableFuture<PendingAckHandle> pendingAckHandleFuture() {
+        return pendingAckHandleCompletableFuture;
+    }
+
+    public void completeHandleFuture() {
+        this.pendingAckHandleCompletableFuture.complete(PendingAckHandleImpl.this);
+    }
+
+    @Override
+    public CompletableFuture<Void> close() {
+        return this.pendingAckStoreFuture.thenAccept(PendingAckStore::closeAsync);
+    }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java
new file mode 100644
index 0000000..352962c
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack.impl;
+
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+/**
+ * The implement of transaction pending ack store.
+ */
+public abstract class PendingAckHandleState {
+
+    /**
+     * The state of the pending ack handle {@link PendingAckHandleState}.
+     */
+    public enum State {
+        None,
+        Initializing,
+        Ready,
+        Close
+    }
+
+    private static final AtomicReferenceFieldUpdater<PendingAckHandleState, State> STATE_UPDATER =
+            AtomicReferenceFieldUpdater.newUpdater(PendingAckHandleState.class, State.class, "state");
+
+    @SuppressWarnings("unused")
+    private volatile State state = null;
+
+    public PendingAckHandleState(State state) {
+        STATE_UPDATER.set(this, state);
+
+    }
+
+    protected boolean changeToReadyState() {
+        return (STATE_UPDATER.compareAndSet(this, State.Initializing, State.Ready));
+    }
+
+    protected boolean changeToInitializingState() {
+        return STATE_UPDATER.compareAndSet(this, State.None, State.Initializing);
+    }
+
+    protected boolean changeToCloseState() {
+        return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Close)
+                || STATE_UPDATER.compareAndSet(this, State.None, State.Close)
+                || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Close));
+    }
+
+    public boolean checkIfReady() {
+        return STATE_UPDATER.get(this) == State.Ready;
+    }
+
+    public State getState() {
+        return STATE_UPDATER.get(this);
+    }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/main/proto/TransactionPendingAck.proto b/pulsar-broker/src/main/proto/TransactionPendingAck.proto
new file mode 100644
index 0000000..fd9bb6a
--- /dev/null
+++ b/pulsar-broker/src/main/proto/TransactionPendingAck.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+syntax = "proto2";
+
+import "pulsar-common/src/main/proto/PulsarApi.proto";
+package pulsar.proto;
+option java_package = "org.apache.pulsar.broker.transaction.pendingack.proto";
+option optimize_for = LITE_RUNTIME;
+
+enum PendingAckOp {
+    ACK    = 1;
+    COMMIT = 2;
+    ABORT  = 3;
+}
+
+message PendingAckMetadata {
+    required uint64 ledgerId = 1;
+    required uint64 entryId  = 2;
+    repeated int64 ack_set = 3;
+    optional int32 batch_size = 4;
+}
+
+message PendingAckMetadataEntry {
+    optional PendingAckOp pending_ack_op = 1;
+    optional pulsar.proto.CommandAck.AckType ack_type = 2;
+    optional uint64 txnid_least_bits = 3;
+    optional uint64 txnid_most_bits = 4;
+    repeated PendingAckMetadata pending_ack_metadata = 5;
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
index 2fa43b7..e3a8161 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
@@ -41,7 +41,6 @@ import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
 import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -574,8 +573,8 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
 
-        InetAddress address1 = InetAddress.getLoopbackAddress();
-        InetAddress address2 = InetAddress.getLocalHost();
+        InetAddress address1 = InetAddress.getByName("127.0.0.1");
+        InetAddress address2 = InetAddress.getByName("0.0.0.0");
         String ipAddress1 = address1.getHostAddress();
         String ipAddress2 = address2.getHostAddress();
 
@@ -748,10 +747,10 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         // 2. duplicate add consumer
         try {
-            sub.addConsumer(consumer);
+            sub.addConsumer(consumer).get();
             fail("Should fail with ConsumerBusyException");
-        } catch (BrokerServiceException e) {
-            assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
 
         // 3. simple remove consumer
@@ -827,9 +826,9 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
             Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3 /* consumer id */, 0,
                     "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(),
                     false /* read compacted */, InitialPosition.Latest, null);
-            addConsumerToSubscription.invoke(topic, sub, consumer3);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub, consumer3)).get();
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
 
@@ -851,9 +850,9 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
             Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5 /* consumer id */, 0,
                     "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(),
                     false /* read compacted */, InitialPosition.Latest, null);
-            addConsumerToSubscription.invoke(topic, sub2, consumer5);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get();
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
     }
@@ -922,9 +921,9 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
             Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0,
                     "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(),
                     false /* read compacted */, InitialPosition.Latest, null);
-            addConsumerToSubscription.invoke(topic, sub, consumer3);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub, consumer3)).get();
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
 
@@ -946,9 +945,9 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
             Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5 /* consumer id */, 0,
                     "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(),
                     false /* read compacted */, InitialPosition.Latest, null);
-            addConsumerToSubscription.invoke(topic, sub2, consumer5);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get();
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
     }
@@ -1007,8 +1006,8 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
         PersistentSubscription sub1 = new PersistentSubscription(topic, "sub1", cursorMock, false);
         PersistentSubscription sub2 = new PersistentSubscription(topic, "sub2", cursorMock, false);
 
-        InetAddress address1 = InetAddress.getLoopbackAddress();
-        InetAddress address2 = InetAddress.getLocalHost();
+        InetAddress address1 = InetAddress.getByName("127.0.0.1");
+        InetAddress address2 = InetAddress.getByName("0.0.0.0");
         String ipAddress1 = address1.getHostAddress();
         String ipAddress2 = address2.getHostAddress();
 
@@ -1026,14 +1025,14 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         // 1. add consumer1 with ipAddress1 to sub1
         Consumer consumer1 = getMockedConsumerWithSpecificAddress(topic, sub1, 1, address1);
-        addConsumerToSubscription.invoke(topic, sub1, consumer1);
+        ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub1, consumer1)).get();
         assertEquals(topic.getNumberOfConsumers(), 1);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 1);
         assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1);
 
         // 2. add consumer2 with ipAddress1 to sub2
         Consumer consumer2 = getMockedConsumerWithSpecificAddress(topic, sub2, 2, address1);
-        addConsumerToSubscription.invoke(topic, sub2, consumer2);
+        ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub2, consumer2)).get();
         assertEquals(topic.getNumberOfConsumers(), 2);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2);
         assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1);
@@ -1041,7 +1040,7 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         // 3. add consumer3 with ipAddress2 to sub1
         Consumer consumer3 = getMockedConsumerWithSpecificAddress(topic, sub1, 3, address2);
-        addConsumerToSubscription.invoke(topic, sub1, consumer3);
+        ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub1, consumer3)).get();
         assertEquals(topic.getNumberOfConsumers(), 3);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 1);
@@ -1050,7 +1049,7 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         // 4. add consumer4 with ipAddress2 to sub2
         Consumer consumer4 = getMockedConsumerWithSpecificAddress(topic, sub2, 4, address2);
-        addConsumerToSubscription.invoke(topic, sub2, consumer4);
+        ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub2, consumer4)).get();
         assertEquals(topic.getNumberOfConsumers(), 4);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2);
         assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 2);
@@ -1060,9 +1059,10 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
         // 5. add consumer5 with ipAddress1 to sub1 but reach maxSameAddressConsumersPerTopic
         try {
             Consumer consumer5 = getMockedConsumerWithSpecificAddress(topic, sub1, 5, address1);
-            addConsumerToSubscription.invoke(topic, sub1, consumer5);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub1, consumer5)).get();
+
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
         assertEquals(topic.getNumberOfConsumers(), 4);
@@ -1072,9 +1072,9 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
         // 6. add consumer6 with ipAddress2 to sub2 but reach maxSameAddressConsumersPerTopic
         try {
             Consumer consumer6 = getMockedConsumerWithSpecificAddress(topic, sub2, 6, address2);
-            addConsumerToSubscription.invoke(topic, sub2, consumer6);
+            ((CompletableFuture<Void>) addConsumerToSubscription.invoke(topic, sub2, consumer6)).get();
             fail("should have failed");
-        } catch (InvocationTargetException e) {
+        } catch (ExecutionException e) {
             assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException);
         }
         assertEquals(topic.getNumberOfConsumers(), 4);
@@ -1122,10 +1122,12 @@ public class PersistentTopicTest extends MockedBookKeeperTestCase {
 
         try {
             Thread.sleep(10); /* delay to ensure that the ubsubscribe gets executed first */
-            new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */,
-                    50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null);
-        } catch (BrokerServiceException e) {
-            assertTrue(e instanceof BrokerServiceException.SubscriptionFencedException);
+            sub.addConsumer(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */,
+                    0, "Cons2"/* consumer name */, 50000, serverCnx,
+                    "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null)).get();
+            fail();
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof BrokerServiceException.SubscriptionFencedException);
         }
     }
 
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java
index 5a17ead..65b6bd9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java
@@ -38,6 +38,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.common.api.proto.CommandAck.AckType;
 import org.apache.pulsar.common.api.proto.MarkersMessageIdData;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.protocol.Markers;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -70,6 +71,7 @@ public class TransactionMarkerDeleteTest extends BrokerTestBase{
         doReturn(configuration).when(pulsarService).getConfig();
         doReturn(true).when(configuration).isTransactionCoordinatorEnabled();
         doReturn(managedLedger).when(topic).getManagedLedger();
+        doReturn(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()).when(topic).getName();
         ManagedCursor cursor = managedLedger.openCursor("test");
         PersistentSubscription persistentSubscription = new PersistentSubscription(topic, "test",
                 managedLedger.openCursor("test"), false);
@@ -100,7 +102,7 @@ public class TransactionMarkerDeleteTest extends BrokerTestBase{
         doReturn(brokerService).when(topic).getBrokerService();
         doReturn(pulsarService).when(brokerService).getPulsar();
         doReturn(configuration).when(pulsarService).getConfig();
-        doReturn(true).when(configuration).isTransactionCoordinatorEnabled();
+        doReturn(false).when(configuration).isTransactionCoordinatorEnabled();
         doReturn(managedLedger).when(topic).getManagedLedger();
         ManagedCursor cursor = managedLedger.openCursor("test");
         PersistentSubscription persistentSubscription = spy(new PersistentSubscription(topic, "test",
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
index bfa678a..ba4e14d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java
@@ -31,6 +31,7 @@ import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -40,7 +41,8 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
@@ -60,7 +62,10 @@ import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.PersistentTopicTest;
 import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider;
-import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider;
+import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider;
+import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl;
+import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleState;
 import org.apache.pulsar.client.api.transaction.TxnID;
 import org.apache.pulsar.common.api.proto.CommandAck.AckType;
 import org.apache.pulsar.common.api.proto.CommandSubscribe;
@@ -71,6 +76,7 @@ import org.apache.pulsar.transaction.common.exception.TransactionConflictExcepti
 import org.apache.pulsar.zookeeper.ZooKeeperCache;
 import org.apache.pulsar.zookeeper.ZooKeeperDataCache;
 import org.apache.zookeeper.ZooKeeper;
+import org.awaitility.Awaitility;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.slf4j.Logger;
@@ -114,6 +120,48 @@ public class PersistentSubscriptionTest {
         svcConfig.setTransactionCoordinatorEnabled(true);
         pulsarMock = spy(new PulsarService(svcConfig));
         doReturn(new InMemTransactionBufferProvider()).when(pulsarMock).getTransactionBufferProvider();
+        doReturn(new TransactionPendingAckStoreProvider() {
+            @Override
+            public CompletableFuture<PendingAckStore> newPendingAckStore(PersistentSubscription subscription) {
+                return CompletableFuture.completedFuture(new PendingAckStore() {
+                    @Override
+                    public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService) {
+                        try {
+                            Field field = PendingAckHandleState.class.getDeclaredField("state");
+                            field.setAccessible(true);
+                            field.set(pendingAckHandle, PendingAckHandleState.State.Ready);
+                        } catch (NoSuchFieldException | IllegalAccessException e) {
+                            fail();
+                        }
+                    }
+
+                    @Override
+                    public CompletableFuture<Void> closeAsync() {
+                        return CompletableFuture.completedFuture(null);
+                    }
+
+                    @Override
+                    public CompletableFuture<Void> appendIndividualAck(TxnID txnID, List<MutablePair<PositionImpl, Integer>> positions) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+
+                    @Override
+                    public CompletableFuture<Void> appendCumulativeAck(TxnID txnID, PositionImpl position) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+
+                    @Override
+                    public CompletableFuture<Void> appendCommitMark(TxnID txnID, AckType ackType) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+
+                    @Override
+                    public CompletableFuture<Void> appendAbortMark(TxnID txnID, AckType ackType) {
+                        return CompletableFuture.completedFuture(null);
+                    }
+                });
+            }
+        }).when(pulsarMock).getTransactionPendingAckStoreProvider();
         doReturn(svcConfig).when(pulsarMock).getConfiguration();
         doReturn(mock(Compactor.class)).when(pulsarMock).getCompactor();
 
@@ -236,8 +284,14 @@ public class PersistentSubscriptionTest {
         }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any());
 
         doReturn(CommandSubscribe.SubType.Exclusive).when(consumerMock).subType();
-
-        persistentSubscription.addConsumer(consumerMock);
+        Awaitility.await().until(() -> {
+            try {
+                persistentSubscription.addConsumer(consumerMock);
+                return true;
+            } catch (Exception e) {
+                return false;
+            }
+        });
 
         // Single ack for txn1
         persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java
index f1dd27f..644ea9e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java
@@ -45,6 +45,7 @@ import org.apache.pulsar.common.api.proto.TxnAction;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.protocol.Commands;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -105,6 +106,9 @@ public class TransactionConsumeTest extends TransactionTestBase {
                 .subscriptionType(SubscriptionType.Shared)
                 .subscribe();
 
+        Awaitility.await().until(exclusiveConsumer::isConnected);
+        Awaitility.await().until(sharedConsumer::isConnected);
+
         long mostSigBits = 2L;
         long leastSigBits = 5L;
         TxnID txnID = new TxnID(mostSigBits, leastSigBits);
@@ -180,6 +184,8 @@ public class TransactionConsumeTest extends TransactionTestBase {
                 .subscriptionName("shared-test")
                 .subscriptionType(SubscriptionType.Shared)
                 .subscribe();
+        Awaitility.await().until(exclusiveConsumer::isConnected);
+        Awaitility.await().until(sharedConsumer::isConnected);
 
         long mostSigBits = 2L;
         long leastSigBits = 5L;
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
index e2836c6..a5bae35 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java
@@ -59,6 +59,7 @@ import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.protocol.Commands;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -279,6 +280,8 @@ public class TransactionProduceTest extends TransactionTestBase {
                 .subscriptionType(SubscriptionType.Shared)
                 .subscribe();
 
+        Awaitility.await().until(consumer::isConnected);
+
         for (int i = 0; i < incomingMessageCnt; i++) {
             Message<byte[]> message = consumer.receive();
             log.info("receive messageId: {}", message.getMessageId());
@@ -340,6 +343,7 @@ public class TransactionProduceTest extends TransactionTestBase {
                 .enableBatchIndexAcknowledgment(true)
                 .subscriptionType(SubscriptionType.Shared)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         for (int i = 0; i < incomingMessageCnt; i++) {
             Message<byte[]> message = consumer.receive();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java
index cf55cf9..db7d1a8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java
@@ -33,6 +33,7 @@ import org.apache.pulsar.broker.transaction.TransactionTestBase;
 import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.SubscriptionType;
@@ -215,9 +216,6 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase {
                 .subscriptionName(subscriptionName)
                 .enableBatchIndexAcknowledgment(true)
                 .subscriptionType(SubscriptionType.Shared)
-                .isAckReceiptEnabled(true)
-                .ackTimeout(2, TimeUnit.SECONDS)
-                .acknowledgmentGroupTime(0, TimeUnit.MICROSECONDS)
                 .subscribe();
 
         @Cleanup
@@ -227,15 +225,12 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase {
                 .batchingMaxMessages(200)
                 .create();
 
-        PersistentSubscription persistentSubscription = null;
         PendingAckHandleImpl pendingAckHandle = null;
 
         LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>> individualAckOfTransaction = null;
         ManagedCursorImpl managedCursor = null;
 
-        ConcurrentSkipListMap<PositionImpl, BitSetRecyclable> batchDeletedIndexes = null;
-
-        Message<byte[]>[] messages = new Message[2];
+        MessageId[] messageIds = new MessageId[2];
         for (int retryCnt = 0; retryCnt < 2; retryCnt++) {
 
             int messageCnt = 1000;
@@ -249,15 +244,20 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase {
             Transaction commitTxn = getTxn();
 
             //send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages
-            for (int i = 0; i < messageCnt - 1; i++) {
+            for (int i = 0; i < messageCnt; i++) {
                 message = consumer.receive(2, TimeUnit.SECONDS);
                 Assert.assertNotNull(message);
-                if (i % 2 == 0) {
-                    consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get();
-                    log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i);
+                // in order to free up 2 position to judge the consumer pending ack delete
+                if (i != 500) {
+                    if (i % 2 == 0) {
+                        consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get();
+                        log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i);
+                    } else {
+                        consumer.acknowledge(message.getMessageId());
+                        log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i);
+                    }
                 } else {
-                    consumer.acknowledge(message.getMessageId());
-                    log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i);
+                    messageIds[retryCnt] = message.getMessageId();
                 }
             }
 
@@ -273,30 +273,54 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase {
                 if (completableFuture != null) {
                     Optional<Topic> topic = completableFuture.get();
                     if (topic.isPresent()) {
-                        persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName);
+                        PersistentSubscription testPersistentSubscription =
+                                (PersistentSubscription) topic.get().getSubscription(subscriptionName);
                         field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
                         field.setAccessible(true);
-                        pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription);
+                        pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription);
                         field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction");
                         field.setAccessible(true);
                         individualAckOfTransaction =
                                 (LinkedMap<TxnID, HashMap<PositionImpl, PositionImpl>>) field.get(pendingAckHandle);
                         assertTrue(individualAckOfTransaction.isEmpty());
-                        managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor();
+                        managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor();
                         field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes");
                         field.setAccessible(true);
-                        batchDeletedIndexes =
+                        final ConcurrentSkipListMap<PositionImpl, BitSetRecyclable> batchDeletedIndexes =
                                 (ConcurrentSkipListMap<PositionImpl, BitSetRecyclable>) field.get(managedCursor);
                         if (retryCnt == 0) {
                             //one message are not ack
+                            Awaitility.await().until(() -> {
+                                return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1;
+                            });
+
                             assertEquals(batchDeletedIndexes.size(), 1);
-                            assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
-                            messages[0] = consumer.receive();
+                            assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
                         } else {
                             //two message are not ack
-                            assertEquals(batchDeletedIndexes.size(), 2);
-                            assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 2);
-                            messages[1] = consumer.receive();
+                            Awaitility.await().until(() -> {
+                                return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2;
+                            });
+
+                            Transaction commitTwice = getTxn();
+
+                            //this message is in one batch point
+                            consumer.acknowledge(messageIds[0]);
+                            Awaitility.await().until(() -> {
+                                return batchDeletedIndexes.size() == 1;
+                            });
+                            assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
+
+                            // this test is for the last message has been cleared in this consumer pending acks
+                            // and it won't clear the last message in cursor batch index ack set
+                            consumer.acknowledgeAsync(messageIds[1], commitTwice).get();
+                            assertEquals(batchDeletedIndexes.size(), 1);
+                            assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
+
+                            // the messages has been produced were all acked, the memory in broker for the messages has been cleared.
+                            commitTwice.commit().get();
+                            assertEquals(batchDeletedIndexes.size(), 0);
+                            assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
                         }
                         count++;
                     }
@@ -304,24 +328,6 @@ public class PendingAckInMemoryDeleteTest extends TransactionTestBase {
             }
             assertEquals(count, 1);
         }
-
-        Transaction commitTwice = getTxn();
-
-        //this message is in one batch point
-        consumer.acknowledge(messages[0].getMessageId());
-        assertEquals(batchDeletedIndexes.size(), 1);
-        assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1);
-
-        // this test is for the last message has been cleared in this consumer pending acks
-        // and it won't clear the last message in cursor batch index ack set
-        consumer.acknowledgeAsync(messages[1].getMessageId(), commitTwice).get();
-        assertEquals(batchDeletedIndexes.size(), 1);
-        assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
-
-        // the messages has been produced were all acked, the memory in broker for the messages has been cleared.
-        commitTwice.commit().get();
-        assertEquals(batchDeletedIndexes.size(), 0);
-        assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0);
     }
 
     private Transaction getTxn() throws Exception {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java
new file mode 100644
index 0000000..da24426
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.transaction.pendingack;
+
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+import com.google.common.collect.Sets;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.transaction.TransactionTestBase;
+import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore;
+import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.api.transaction.Transaction;
+import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Test for consuming transaction messages.
+ */
+@Slf4j
+public class PendingAckPersistentTest extends TransactionTestBase {
+
+    private final static String PENDING_ACK_REPLAY_TOPIC = "persistent://public/txn/pending-ack-replay";
+
+    @BeforeMethod
+    public void setup() throws Exception {
+        setBrokerCount(1);
+        super.internalSetup();
+
+        String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":");
+        String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1];
+        admin.clusters().createCluster(CLUSTER_NAME, new ClusterData("http://localhost:" + webServicePort));
+        admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(),
+                new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME)));
+        admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString());
+        admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16);
+        admin.tenants().createTenant("public",
+                new TenantInfo(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME)));
+        admin.namespaces().createNamespace("public/txn", 10);
+        admin.topics().createNonPartitionedTopic(PENDING_ACK_REPLAY_TOPIC);
+
+        pulsarClient = PulsarClient.builder()
+                .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl())
+                .statsInterval(0, TimeUnit.SECONDS)
+                .enableTransaction(true)
+                .build();
+
+        Thread.sleep(1000 * 3);
+    }
+
+    @AfterMethod(alwaysRun = true)
+    protected void cleanup() {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void individualPendingAckReplayTest() throws Exception {
+        int messageCount = 1000;
+        String subName = "individual-test";
+
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer()
+                .topic(PENDING_ACK_REPLAY_TOPIC)
+                .enableBatching(true)
+                .batchingMaxMessages(200)
+                .create();
+
+        @Cleanup
+        Consumer<byte[]> consumer = pulsarClient.newConsumer()
+                .topic(PENDING_ACK_REPLAY_TOPIC)
+                .subscriptionName(subName)
+                .subscriptionType(SubscriptionType.Shared)
+                .enableBatchIndexAcknowledgment(true)
+                .subscribe();
+
+        Transaction abortTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        List<MessageId> pendingAckMessageIds = new ArrayList<>();
+        List<MessageId> normalAckMessageIds = new ArrayList<>();
+        for (int i = 0; i < messageCount; i++) {
+            producer.send("Hello Pulsar!".getBytes());
+            Message<byte[]> message = consumer.receive();
+            if (i % 2 == 0) {
+                consumer.acknowledgeAsync(message.getMessageId(), abortTxn).get();
+                pendingAckMessageIds.add(message.getMessageId());
+            } else {
+                normalAckMessageIds.add(message.getMessageId());
+            }
+        }
+
+        //in order to test pending ack replay
+        admin.topics().unload(PENDING_ACK_REPLAY_TOPIC);
+        Awaitility.await().until(consumer::isConnected);
+        Transaction commitTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        Transaction txn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        // this messageIds are ack by transaction
+        for (int i = 0; i < pendingAckMessageIds.size(); i++) {
+            try {
+                consumer.acknowledgeAsync(pendingAckMessageIds.get(i), txn).get();
+                fail();
+            } catch (ExecutionException e) {
+                assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException);
+            }
+        }
+        // this messageIds are not ack by transaction
+        for (int i = 0; i < normalAckMessageIds.size(); i++) {
+            consumer.acknowledgeAsync(normalAckMessageIds.get(i), commitTxn).get();
+        }
+
+        txn.abort().get();
+        // commit this txn , normalAckMessageIds are in pending ack state
+        commitTxn.commit().get();
+        // abort this txn, pendingAckMessageIds are delete from pending ack state
+        abortTxn.abort().get();
+
+        // replay this pending ack
+        admin.topics().unload(PENDING_ACK_REPLAY_TOPIC);
+        Awaitility.await().until(consumer::isConnected);
+
+        abortTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        commitTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+        // normalAckMessageIds are ack and then commit, so ack fail
+        for (int i = 0; i < normalAckMessageIds.size(); i++) {
+            try {
+                consumer.acknowledgeAsync(normalAckMessageIds.get(i), abortTxn).get();
+                fail();
+            } catch (ExecutionException e) {
+                assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException);
+            }
+        }
+
+        // pendingAckMessageIds are all abort, so can ack again
+        for (int i = 0; i < pendingAckMessageIds.size(); i++) {
+            consumer.acknowledgeAsync(pendingAckMessageIds.get(i), commitTxn).get();
+        }
+
+        abortTxn.abort().get();
+        commitTxn.commit().get();
+
+        PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService()
+                .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get();
+        Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
+        field.setAccessible(true);
+        PendingAckHandleImpl pendingAckHandle =
+                (PendingAckHandleImpl) field.get(topic.getSubscription(subName));
+        field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture");
+        field.setAccessible(true);
+        CompletableFuture<PendingAckStore> pendingAckStoreCompletableFuture =
+                (CompletableFuture<PendingAckStore>) field.get(pendingAckHandle);
+        pendingAckStoreCompletableFuture.get();
+
+        field = MLPendingAckStore.class.getDeclaredField("cursor");
+        field.setAccessible(true);
+
+        ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get());
+
+        // in order to check out the pending ack cursor is clear whether or not.
+        Awaitility.await()
+                .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition())
+                        .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1);
+    }
+
+    @Test
+    public void cumulativePendingAckReplayTest() throws Exception {
+        int messageCount = 1000;
+        String subName = "cumulative-test";
+
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer()
+                .topic(PENDING_ACK_REPLAY_TOPIC)
+                .enableBatching(true)
+                .batchingMaxMessages(200)
+                .create();
+
+        @Cleanup
+        Consumer<byte[]> consumer = pulsarClient.newConsumer()
+                .topic(PENDING_ACK_REPLAY_TOPIC)
+                .subscriptionName(subName)
+                .subscriptionType(SubscriptionType.Failover)
+                .enableBatchIndexAcknowledgment(true)
+                .subscribe();
+
+        Transaction abortTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        List<MessageId> pendingAckMessageIds = new ArrayList<>();
+        for (int i = 0; i < messageCount; i++) {
+            producer.send("Hello Pulsar!".getBytes());
+        }
+
+        for (int i = 0; i < messageCount; i++) {
+            Message<byte[]> message = consumer.receive();
+            pendingAckMessageIds.add(message.getMessageId());
+            consumer.acknowledgeCumulativeAsync(message.getMessageId(), abortTxn).get();
+        }
+
+        admin.topics().unload(PENDING_ACK_REPLAY_TOPIC);
+        Transaction txn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+
+        Awaitility.await().until(consumer::isConnected);
+
+        for (int i = 0; i < pendingAckMessageIds.size(); i++) {
+            try {
+                consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get();
+                fail();
+            } catch (ExecutionException e) {
+                assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException);
+            }
+        }
+        Transaction commitTxn = pulsarClient.newTransaction()
+                .withTransactionTimeout(30, TimeUnit.SECONDS).build().get();
+        abortTxn.abort().get();
+
+        for (int i = 0; i < pendingAckMessageIds.size(); i++) {
+            consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), commitTxn).get();
+        }
+        commitTxn.commit().get();
+
+        admin.topics().unload(PENDING_ACK_REPLAY_TOPIC);
+        Awaitility.await().until(consumer::isConnected);
+
+        for (int i = 0; i < pendingAckMessageIds.size(); i++) {
+            try {
+                consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get();
+                fail();
+            } catch (ExecutionException e) {
+                assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException);
+            }
+        }
+
+        PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService()
+                .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get();
+        Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle");
+        field.setAccessible(true);
+        PendingAckHandleImpl pendingAckHandle =
+                (PendingAckHandleImpl) field.get(topic.getSubscription(subName));
+        field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture");
+        field.setAccessible(true);
+        CompletableFuture<PendingAckStore> pendingAckStoreCompletableFuture =
+                (CompletableFuture<PendingAckStore>) field.get(pendingAckHandle);
+        pendingAckStoreCompletableFuture.get();
+
+        field = MLPendingAckStore.class.getDeclaredField("cursor");
+        field.setAccessible(true);
+
+        ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get());
+
+        // in order to check out the pending ack cursor is clear whether or not.
+        Awaitility.await()
+                .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition())
+                        .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1);
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
index 1053efd..58d87f6 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java
@@ -46,7 +46,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
 import org.apache.pulsar.broker.transaction.TransactionTestBase;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.ProducerBuilder;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -142,6 +141,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .subscriptionName("test")
                 .enableBatchIndexAcknowledgment(true)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         ProducerBuilder<byte[]> producerBuilder = pulsarClient
                 .newProducer()
@@ -215,6 +215,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .subscriptionName(subName)
                 .enableBatchIndexAcknowledgment(true)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         // Can't receive transaction messages before abort.
         Message<byte[]> message = consumer.receive(2, TimeUnit.SECONDS);
@@ -305,6 +306,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .enableBatchIndexAcknowledgment(true)
                 .subscriptionType(subscriptionType)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         @Cleanup
         Producer<byte[]> producer = pulsarClient.newProducer()
@@ -380,6 +382,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .enableBatchIndexAcknowledgment(true)
                 .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         @Cleanup
         Producer<byte[]> producer = pulsarClient
@@ -503,6 +506,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .subscriptionType(subscriptionType)
                 .ackTimeout(1, TimeUnit.MINUTES)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         @Cleanup
         Producer<byte[]> producer = pulsarClient.newProducer()
@@ -649,6 +653,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .subscriptionName("test")
                 .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         for (int i = 0; i < txnCnt * messageCnt; i++) {
             Message<byte[]> message = consumer.receive();
@@ -665,6 +670,7 @@ public class TransactionEndToEndTest extends TransactionTestBase {
                 .topic(topic)
                 .subscriptionName("test")
                 .subscribe();
+        Awaitility.await().until(consumer::isConnected);
 
         @Cleanup
         Producer<byte[]> producer = pulsarClient.newProducer()
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java
index 3cc05be..20110cd 100644
--- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java
+++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java
@@ -30,6 +30,7 @@ import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.client.api.transaction.Transaction;
+import org.awaitility.Awaitility;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
@@ -99,6 +100,7 @@ public class TransactionTest extends TransactionTestBase {
                 .subscriptionType(SubscriptionType.Shared)
                 .enableBatchIndexAcknowledgment(true)
                 .subscribe();
+        Awaitility.await().until(transferConsumer::isConnected);
         log.info("transfer consumer create finished");
 
         @Cleanup
@@ -114,6 +116,7 @@ public class TransactionTest extends TransactionTestBase {
                 .subscriptionName("integration-test")
                 .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                 .subscribe();
+        Awaitility.await().until(balanceUpdateConsumer::isConnected);
         log.info("balance update consumer create finished");
 
         while (true) {