You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2025/06/19 17:08:09 UTC
(pulsar) 03/03: [fix][txn] Fix deadlock when loading transaction buffer snapshot (#24401)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 27e27d36712424b392cdc9eb60eda04652cffbb9
Author: Yunze Xu <xy...@163.com>
AuthorDate: Wed Jun 11 19:45:44 2025 +0800
[fix][txn] Fix deadlock when loading transaction buffer snapshot (#24401)
(cherry picked from commit 1b7e4a7bbd6a8e6a309a87a9c3ad19bbe837e7c0)
---
.../org/apache/pulsar/broker/PulsarService.java | 7 ++++++
.../SingleSnapshotAbortedTxnProcessorImpl.java | 2 +-
.../SnapshotSegmentAbortedTxnProcessorImpl.java | 2 +-
.../broker/transaction/buffer/impl/TableView.java | 10 +++++++++
.../java/org/apache/pulsar/utils/SimpleCache.java | 16 +++++++++++++-
.../pulsar/broker/transaction/TransactionTest.java | 5 ++++-
.../apache/pulsar/client/impl/ConsumerImpl.java | 25 ++++++++++++++++------
7 files changed, 57 insertions(+), 10 deletions(-)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index f7325653305..66e5b495076 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -275,6 +275,7 @@ public class PulsarService implements AutoCloseable, ShutdownService {
private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider;
private final ExecutorProvider transactionExecutorProvider;
+ private final ExecutorProvider transactionSnapshotRecoverExecutorProvider;
private String brokerId;
private final CompletableFuture<Void> readyForIncomingRequestsFuture = new CompletableFuture<>();
private final List<Runnable> pendingTasksBeforeReadyForIncomingRequests = new ArrayList<>();
@@ -337,8 +338,11 @@ public class PulsarService implements AutoCloseable, ShutdownService {
if (config.isTransactionCoordinatorEnabled()) {
this.transactionExecutorProvider = new ExecutorProvider(this.getConfiguration()
.getNumTransactionReplayThreadPoolSize(), "pulsar-transaction-executor");
+ this.transactionSnapshotRecoverExecutorProvider = new ExecutorProvider(this.getConfiguration()
+ .getNumTransactionReplayThreadPoolSize(), "pulsar-transaction-snapshot-recover");
} else {
this.transactionExecutorProvider = null;
+ this.transactionSnapshotRecoverExecutorProvider = null;
}
this.ioEventLoopGroup = EventLoopUtil.newEventLoopGroup(config.getNumIOThreads(), config.isEnableBusyWait(),
@@ -600,6 +604,9 @@ public class PulsarService implements AutoCloseable, ShutdownService {
if (transactionExecutorProvider != null) {
transactionExecutorProvider.shutdownNow();
}
+ if (transactionSnapshotRecoverExecutorProvider != null) {
+ transactionSnapshotRecoverExecutorProvider.shutdownNow();
+ }
if (transactionTimer != null) {
transactionTimer.stop();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
index 0a775dcee03..616f6e4e739 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java
@@ -86,7 +86,7 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso
public CompletableFuture<PositionImpl> recoverFromSnapshot() {
final var future = new CompletableFuture<PositionImpl>();
final var pulsar = topic.getBrokerService().getPulsar();
- pulsar.getTransactionExecutorProvider().getExecutor(this).execute(() -> {
+ pulsar.getTransactionSnapshotRecoverExecutorProvider().getExecutor(this).execute(() -> {
try {
final var snapshot = pulsar.getTransactionBufferSnapshotServiceFactory().getTxnBufferSnapshotService()
.getTableView().readLatest(topic.getName());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
index fb47943035a..55e4766ab4b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java
@@ -223,7 +223,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess
public CompletableFuture<PositionImpl> recoverFromSnapshot() {
final var pulsar = topic.getBrokerService().getPulsar();
final var future = new CompletableFuture<PositionImpl>();
- pulsar.getTransactionExecutorProvider().getExecutor(this).execute(() -> {
+ pulsar.getTransactionSnapshotRecoverExecutorProvider().getExecutor(this).execute(() -> {
try {
final var indexes = pulsar.getTransactionBufferSnapshotServiceFactory()
.getTxnBufferSnapshotIndexService().getTableView().readLatest(topic.getName());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java
index 7608a393cc9..31584f4e5e5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TableView.java
@@ -58,6 +58,16 @@ public class TableView<T> {
}
public T readLatest(String topic) throws Exception {
+ try {
+ return internalReadLatest(topic);
+ } catch (Exception e) {
+ final var namespace = TopicName.get(topic).getNamespaceObject();
+ readers.remove(namespace);
+ throw e;
+ }
+ }
+
+ private T internalReadLatest(String topic) throws Exception {
final var reader = getReader(topic);
while (wait(reader.hasMoreEventsAsync(), "has more events")) {
final var msg = wait(reader.readNextAsync(), "read message");
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java
index 6a3a6721198..6bd706068d0 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/SimpleCache.java
@@ -41,7 +41,7 @@ public class SimpleCache<K, V> {
boolean tryExpire() {
if (System.currentTimeMillis() >= deadlineMs) {
- expireCallback.accept(value);
+ cancel();
return true;
} else {
return false;
@@ -51,6 +51,10 @@ public class SimpleCache<K, V> {
void updateDeadline() {
deadlineMs = System.currentTimeMillis() + timeoutMs;
}
+
+ void cancel() {
+ expireCallback.accept(value);
+ }
}
public SimpleCache(final ScheduledExecutorService scheduler, final long timeoutMs, final long frequencyMs) {
@@ -80,4 +84,14 @@ public class SimpleCache<K, V> {
cache.put(key, newValue);
return newValue.value;
}
+
+ public void remove(final K key) {
+ final ExpirableValue<V> value;
+ synchronized (this) {
+ value = cache.remove(key);
+ }
+ if (value != null) {
+ value.cancel();
+ }
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
index d6e3c20543a..9d75c722cec 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java
@@ -169,7 +169,10 @@ public class TransactionTest extends TransactionTestBase {
@BeforeClass
protected void setup() throws Exception {
- setUpBase(NUM_BROKERS, NUM_PARTITIONS, NAMESPACE1 + "/test", 0);
+ // Use a single transaction thread to reproduce possible deadlock easily
+ conf.setNumTransactionReplayThreadPoolSize(1);
+ conf.setManagedLedgerNumSchedulerThreads(1);
+ setUpBase(NUM_BROKERS, NUM_PARTITIONS, NAMESPACE1 + "/test", 0);
}
@AfterClass(alwaysRun = true)
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 9705331d7f4..5792d785ff5 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -222,6 +222,8 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
private final AtomicReference<ClientCnx> clientCnxUsedForConsumerRegistration = new AtomicReference<>();
private final List<Throwable> previousExceptions = new CopyOnWriteArrayList<Throwable>();
private volatile boolean hasSoughtByTimestamp = false;
+ // This field will be set after the state becomes Failed, then the following operations will fail immediately
+ private volatile Throwable failReason = null;
static <T> ConsumerImpl<T> newConsumerImpl(PulsarClientImpl client,
String topic,
@@ -921,11 +923,13 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
} else if (!subscribeFuture.isDone()) {
// unable to create new consumer, fail operation
setState(State.Failed);
+ final Throwable throwable = PulsarClientException.wrap(e, String.format("Failed to subscribe the "
+ + "topic %s with subscription name %s when connecting to the broker", topicName.toString(),
+ subscription));
+ fail(throwable);
+
closeConsumerTasks();
- subscribeFuture.completeExceptionally(
- PulsarClientException.wrap(e, String.format("Failed to subscribe the topic %s "
- + "with subscription name %s when connecting to the broker",
- topicName.toString(), subscription)));
+ subscribeFuture.completeExceptionally(throwable);
client.cleanupConsumer(this);
} else if (isUnrecoverableError(e.getCause())) {
closeWhenReceivedUnrecoverableError(e.getCause(), cnx);
@@ -966,7 +970,7 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
topic, subscription, cnxStr, t.getClass().getName(), t.getMessage());
closeAsync().whenComplete((__, ex) -> {
if (ex == null) {
- setState(State.Failed);
+ fail(t);
return;
}
log.error("[{}][{}] {} Failed to close consumer after got an error that does not support to retry: {} {}",
@@ -1060,7 +1064,7 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
if (nonRetriableError || timeout) {
exception.setPreviousExceptions(previousExceptions);
if (subscribeFuture.completeExceptionally(exception)) {
- setState(State.Failed);
+ fail(exception);
if (nonRetriableError) {
log.info("[{}] Consumer creation failed for consumer {} with unretriableError {}",
topic, consumerId, exception.getMessage());
@@ -2720,6 +2724,10 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
return null;
});
} else {
+ if (failReason != null) {
+ future.completeExceptionally(failReason);
+ return;
+ }
long nextDelay = Math.min(backoff.next(), remainingTime.get());
if (nextDelay <= 0) {
future.completeExceptionally(
@@ -3135,4 +3143,9 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
IN_PROGRESS,
COMPLETED
}
+
+ private void fail(Throwable throwable) {
+ setState(State.Failed);
+ failReason = throwable;
+ }
}