You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/21 10:19:47 UTC

(pulsar) branch master updated: [fix][broker] Retry incompatible replication schemas without topic unload (#27026)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new f530dfd030bd [fix][broker] Retry incompatible replication schemas without topic unload (#27026)
f530dfd030bd is described below

commit f530dfd030bdb7464d94eb70c33f6c9a5447e5e2
Author: Penghui Li <pe...@apache.org>
AuthorDate: Wed Oct 21 03:19:40 2026 -0700

    [fix][broker] Retry incompatible replication schemas without topic unload (#27026)
---
 .../broker/service/OneWayReplicatorTest.java       | 101 ++++++
 .../apache/pulsar/client/impl/ProducerImpl.java    | 149 +++++----
 .../pulsar/client/impl/ProducerImplTest.java       | 344 +++++++++++++++++++++
 3 files changed, 538 insertions(+), 56 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
index ac20c2eb9dde..dadedc0b932a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
@@ -86,6 +86,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.logging.log4j.Level;
 import org.apache.pulsar.broker.BrokerTestUtil;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.resources.ClusterResources;
@@ -110,6 +111,9 @@ import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.client.api.schema.GenericRecord;
+import org.apache.pulsar.client.api.schema.GenericSchema;
+import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
+import org.apache.pulsar.client.api.schema.SchemaBuilder;
 import org.apache.pulsar.client.impl.ClientBuilderImpl;
 import org.apache.pulsar.client.impl.ClientCnx;
 import org.apache.pulsar.client.impl.ClientImplInternalSetter;
@@ -136,11 +140,13 @@ import org.apache.pulsar.common.policies.data.TenantInfo;
 import org.apache.pulsar.common.policies.data.TopicStats;
 import org.apache.pulsar.common.policies.data.TopicType;
 import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
+import org.apache.pulsar.common.schema.KeyValue;
 import org.apache.pulsar.common.schema.LongSchemaVersion;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.apache.pulsar.utils.TestLogAppender;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.apache.pulsar.zookeeper.ZookeeperServerTest;
 import org.awaitility.Awaitility;
@@ -2133,6 +2139,101 @@ public class OneWayReplicatorTest extends OneWayReplicatorTestBase {
         };
     }
 
+    @DataProvider
+    public Object[][] schemaRecovery() {
+        return new Object[][] {{false, false}, {false, true}, {true, false}, {true, true}};
+    }
+
+    @Test(dataProvider = "schemaRecovery", timeOut = 120_000)
+    public void testReplicationResumesAfterSchemaConflictResolved(boolean deduplication, boolean deleteSchema)
+            throws Exception {
+        String topicName = BrokerTestUtil.newUniqueName("persistent://"
+                + sourceClusterAlwaysSchemaCompatibleNamespace + "/schema-retry-");
+        RecordSchemaBuilder oldKeyBuilder = SchemaBuilder.record("Key");
+        oldKeyBuilder.field("id").type(SchemaType.INT64);
+        RecordSchemaBuilder newKeyBuilder = SchemaBuilder.record("Key");
+        newKeyBuilder.field("game_id").type(SchemaType.INT64);
+        GenericSchema<GenericRecord> oldKey = Schema.generic(oldKeyBuilder.build(SchemaType.AVRO));
+        GenericSchema<GenericRecord> newKey = Schema.generic(newKeyBuilder.build(SchemaType.AVRO));
+        Schema<KeyValue<GenericRecord, String>> schema = Schema.KeyValue(newKey, Schema.STRING);
+
+        admin2.topics().createNonPartitionedTopic(topicName);
+        admin2.topicPolicies().setSchemaCompatibilityStrategy(topicName, SchemaCompatibilityStrategy.FULL);
+        admin2.topicPolicies().setDeduplicationStatus(topicName, deduplication);
+        admin2.schemas().createSchema(topicName, Schema.KeyValue(oldKey, Schema.STRING).getSchemaInfo());
+        try (Consumer<byte[]> consumer = client2.newConsumer(Schema.BYTES).topic(topicName)
+                .subscriptionName("schema-retry").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+                .subscribe();
+             Producer<byte[]> producer = client1.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName)
+                     .enableBatching(false).create();
+             TestLogAppender logs = TestLogAppender.create(ProducerImpl.class)) {
+            waitReplicatorStarted(topicName);
+            PersistentTopic destination = (PersistentTopic) broker2.getTopic(topicName, false).join().orElseThrow();
+            Awaitility.await().untilAsserted(() -> {
+                assertThat(destination.getSchemaCompatibilityStrategy()).isEqualTo(SchemaCompatibilityStrategy.FULL);
+                assertThat(destination.getMessageDeduplication().isEnabled()).isEqualTo(deduplication);
+            });
+            PersistentTopic source = (PersistentTopic) broker1.getTopic(topicName, false).join().orElseThrow();
+            PersistentReplicator replicator = (PersistentReplicator) source.getReplicators().get(cluster2);
+            ProducerImpl<?> replicationProducer = replicator.getProducer();
+            ClientCnx connection = replicationProducer.getClientCnx();
+            long producerEpoch = replicationProducer.getConnectionHandler().getEpoch();
+            List<byte[]> expected = new ArrayList<>();
+            for (int i = 0; i < 10; i++) {
+                KeyValue<GenericRecord, String> value = new KeyValue<>(
+                        newKey.newRecordBuilder().set("game_id", (long) i).build(), "game-" + i);
+                producer.newMessage(schema).value(value).send();
+                expected.add(schema.encode(value));
+                // A schema-free message must not overtake the incompatible message.
+                byte[] following = ("after-" + i).getBytes(StandardCharsets.UTF_8);
+                producer.newMessage(Schema.BYTES).value(following).send();
+                expected.add(following);
+            }
+            Awaitility.await().untilAsserted(() -> assertThat(logs.getEvents()).anySatisfy(event ->
+                    assertThat(event.getMessage().getFormattedMessage()).contains("Publishing paused")));
+            assertThat(replicator.getCursor().getNumberOfEntriesInBacklog(false)).isEqualTo(expected.size());
+            assertThat(consumer.receive(1, TimeUnit.SECONDS)).isNull();
+            assertThat(logs.getEvents().stream().filter(event -> event.getLevel() == Level.WARN
+                    && event.getMessage().getFormattedMessage().contains("GetOrCreateSchema error")).count())
+                    .isEqualTo(1);
+            assertThat(logs.getEvents()).noneSatisfy(event -> {
+                assertThat(event.getLevel()).isEqualTo(Level.INFO);
+                assertThat(event.getMessage().getFormattedMessage()).contains("GetOrCreateSchema request");
+            });
+
+            if (deleteSchema) {
+                admin2.schemas().deleteSchema(topicName);
+            } else {
+                admin2.topicPolicies().setSchemaCompatibilityStrategy(topicName,
+                        SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE);
+            }
+            Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() ->
+                    assertThat(replicator.getCursor().getNumberOfEntriesInBacklog(false)).isZero());
+            assertThat(admin2.schemas().getSchemaInfo(topicName).getSchema())
+                    .isEqualTo(schema.getSchemaInfo().getSchema());
+            assertThat(replicator.getProducer()).isSameAs(replicationProducer);
+            assertThat(replicationProducer.getClientCnx()).isSameAs(connection);
+            assertThat(replicationProducer.getConnectionHandler().getEpoch()).isEqualTo(producerEpoch);
+            assertThat(broker1.getTopic(topicName, false).join().orElseThrow()).isSameAs(source);
+            assertThat(broker2.getTopic(topicName, false).join().orElseThrow()).isSameAs(destination);
+            for (byte[] payload : expected) {
+                Message<byte[]> received = consumer.receive(5, TimeUnit.SECONDS);
+                assertThat(received).isNotNull();
+                assertThat(received.getData()).isEqualTo(payload);
+                consumer.acknowledge(received);
+            }
+            assertThat(consumer.receive(1, TimeUnit.SECONDS)).isNull();
+            producer.newMessage(schema).value(new KeyValue<>(
+                    newKey.newRecordBuilder().set("game_id", 100L).build(), "live")).send();
+            assertThat(consumer.receive(5, TimeUnit.SECONDS)).isNotNull();
+        } finally {
+            admin1.topics().setReplicationClusters(topicName, Collections.singletonList(cluster1));
+            waitReplicatorStopped(topicName, false);
+            admin1.topics().delete(topicName, true);
+            admin2.topics().delete(topicName, true);
+        }
+    }
+
     @Test(dataProvider = "enableDeduplication")
     public void testIncompatibleMultiVersionSchema(boolean enableDeduplication) throws Exception {
         final String topicName = BrokerTestUtil.newUniqueName("persistent://"
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
index 84e2042ea840..3bbdee956b47 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
@@ -201,6 +201,11 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
     private final Counter producersOpenedCounter;
     private final Counter producersClosedCounter;
     private final boolean pauseSendingToPreservePublishOrderOnSchemaRegFailure;
+    private final Backoff schemaRegistrationBackoff;
+    // Guarded by the producer lock, including schema callbacks and retry tasks.
+    private long schemaRegistrationGeneration;
+    private ScheduledFuture<?> schemaRegistrationRetryTask;
+    private boolean schemaRegistrationFailed;
     // This variable can be exposed as a metrics in the future, a PIP is needed.
     private final AtomicInteger pendingQueueFullCounter;
 
@@ -218,6 +223,10 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
         // Otherwise, it may cause an out-of-order issue, and it may lead to a messages lost issue if users enabled
         // deduplication on the remote side.
         this.pauseSendingToPreservePublishOrderOnSchemaRegFailure = conf.isReplProducer();
+        this.schemaRegistrationBackoff = Backoff.builder()
+                .initialDelay(Duration.ofNanos(client.getConfiguration().getInitialBackoffIntervalNanos()))
+                .maxBackoff(Duration.ofNanos(client.getConfiguration().getMaxBackoffIntervalNanos()))
+                .build();
         if (conf.getMaxPendingMessages() > 0) {
             // With admission done upstream the permit is never taken but would still be released.
             checkArgument(!conf.isMemoryLimitAdmittedUpstream(),
@@ -982,53 +991,85 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
         return true;
     }
 
-    private void tryRegisterSchema(ClientCnx cnx, final MessageImpl<?> msg, SendCallback callback,
-                                    long expectedCnxEpoch) {
+    private void tryRegisterSchema(ClientCnx cnx, OpSendMsg op, long expectedCnxEpoch) {
         if (!changeToRegisteringSchemaState()) {
             return;
         }
+        long generation = ++schemaRegistrationGeneration;
+        cancelSchemaRegistrationRetry();
+        MessageImpl<?> msg = op.msg;
+        long sequenceId = op.sequenceId;
         SchemaInfo schemaInfo = msg.hasReplicateFrom() ? msg.getSchemaInfoForReplicator() : msg.getSchemaInfo();
         schemaInfo = Optional.ofNullable(schemaInfo)
                 .filter(si -> si.getType().getValue() > 0)
                 .orElse(Schema.BYTES.getSchemaInfo());
-        getOrCreateSchemaAsync(cnx, schemaInfo).handle((v, ex) -> {
-            if (ex != null) {
-                Throwable t = FutureUtil.unwrapCompletionException(ex);
-                log.warn()
-                        .exception(t)
-                        .log("GetOrCreateSchema error");
-                if (t instanceof PulsarClientException.IncompatibleSchemaException) {
-                    // Only the first time of failed schema registration will trigger a "recoverProcessOpSendMsgFrom".
-                    if (!Broken.equals(msg.getSchemaState())) {
-                        cnx.ctx().channel().eventLoop().execute(() -> {
-                            synchronized (ProducerImpl.this) {
+        getOrCreateSchemaAsync(cnx, schemaInfo).whenComplete((v, ex) ->
+                cnx.ctx().channel().eventLoop().execute(() -> {
+                    synchronized (ProducerImpl.this) {
+                        if (generation != schemaRegistrationGeneration || cnx() != cnx
+                                || expectedCnxEpoch != connectionHandler.getEpoch()
+                                || getState() != State.RegisteringSchema) {
+                            return;
+                        }
+                        // A send timeout can dispose and recycle the operation while registration is outstanding.
+                        if (op.msg != msg || op.sequenceId != sequenceId) {
+                            recoverProcessOpSendMsgFrom(cnx, null, false, expectedCnxEpoch);
+                            return;
+                        }
+                        if (ex == null) {
+                            log.debug("GetOrCreateSchema succeed");
+                            // An empty version means there is no schema and must not be set on message metadata.
+                            if (v.length != 0) {
+                                schemaCache.putIfAbsent(msg.getSchemaHash(), v);
+                                msg.getMessageBuilder().setSchemaVersion(v);
+                            } else {
+                                schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes());
+                            }
+                            msg.setSchemaState(MessageImpl.SchemaState.Ready);
+                            schemaRegistrationBackoff.reset();
+                            schemaRegistrationFailed = false;
+                            recoverProcessOpSendMsgFrom(cnx, msg, false, expectedCnxEpoch);
+                            return;
+                        }
+                        Throwable t = FutureUtil.unwrapCompletionException(ex);
+                        if (schemaRegistrationFailed) {
+                            log.debug().exceptionMessage(t).log("GetOrCreateSchema retry failed");
+                        } else {
+                            log.warn().exception(t).log("GetOrCreateSchema error");
+                        }
+                        schemaRegistrationFailed = true;
+                        if (t instanceof IncompatibleSchemaException) {
+                            if (!pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
+                                schemaRegistrationBackoff.reset();
+                                schemaRegistrationFailed = false;
                                 recoverProcessOpSendMsgFrom(cnx, msg, true, expectedCnxEpoch);
+                                return;
                             }
-                        });
+                            recoverProcessOpSendMsgFrom(cnx, msg, true, expectedCnxEpoch);
+                        }
+                        schemaRegistrationRetryTask = cnx.ctx().channel().eventLoop().schedule(() -> {
+                            synchronized (ProducerImpl.this) {
+                                if (generation != schemaRegistrationGeneration) {
+                                    return;
+                                }
+                                schemaRegistrationRetryTask = null;
+                                if (cnx() == cnx && expectedCnxEpoch == connectionHandler.getEpoch()
+                                        && getState() == State.RegisteringSchema) {
+                                    MessageImpl<?> retryFrom =
+                                            op.msg == msg && op.sequenceId == sequenceId ? msg : null;
+                                    recoverProcessOpSendMsgFrom(cnx, retryFrom, false, expectedCnxEpoch);
+                                }
+                            }
+                        }, schemaRegistrationBackoff.next().toMillis(), TimeUnit.MILLISECONDS);
                     }
-                    return null;
-                }
-            } else {
-                log.info("GetOrCreateSchema succeed");
-                // In broker, if schema version is an empty byte array, it means the topic doesn't have schema.
-                // In this case, we cache the schema version to `SchemaVersion.Empty.bytes()`.
-                // When we need to set the schema version of the message metadata,
-                // we should check if the cached schema version is `SchemaVersion.Empty.bytes()`
-                if (v.length != 0) {
-                    schemaCache.putIfAbsent(msg.getSchemaHash(), v);
-                    msg.getMessageBuilder().setSchemaVersion(v);
-                } else {
-                    schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes());
-                }
-                msg.setSchemaState(MessageImpl.SchemaState.Ready);
-            }
-            cnx.ctx().channel().eventLoop().execute(() -> {
-                synchronized (ProducerImpl.this) {
-                    recoverProcessOpSendMsgFrom(cnx, msg, false, expectedCnxEpoch);
-                }
-            });
-            return null;
-        });
+                }));
+    }
+
+    private void cancelSchemaRegistrationRetry() {
+        if (schemaRegistrationRetryTask != null) {
+            schemaRegistrationRetryTask.cancel(false);
+            schemaRegistrationRetryTask = null;
+        }
     }
 
     private CompletableFuture<byte[]> getOrCreateSchemaAsync(ClientCnx cnx, SchemaInfo schemaInfo) {
@@ -1041,7 +1082,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
         }
         long requestId = client.newRequestId();
         ByteBuf request = Commands.newGetOrCreateSchema(requestId, topic, producerName, schemaInfo);
-        log.info("GetOrCreateSchema request");
+        log.debug("GetOrCreateSchema request");
         return cnx.sendGetOrCreateSchema(request, requestId);
     }
 
@@ -2401,7 +2442,9 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
         return true;
     }
 
-    private void closeProducerTasks() {
+    private synchronized void closeProducerTasks() {
+        schemaRegistrationGeneration++;
+        cancelSchemaRegistrationRetry();
         Timeout timeout = sendTimeout;
         if (timeout != null) {
             timeout.cancel();
@@ -2811,7 +2854,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
             final ClientCnx cnx = getCnxIfReady();
             if (cnx != null) {
                 if (op.msg != null && op.msg.getSchemaState() == None) {
-                    tryRegisterSchema(cnx, op.msg, op.callback, this.connectionHandler.getEpoch());
+                    tryRegisterSchema(cnx, op, this.connectionHandler.getEpoch());
                     return;
                 }
                 // If we do have a connection, the message is sent immediately, otherwise we'll try again once a new
@@ -2886,10 +2929,11 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
      * 3. If using multiple version producer, the new schema was failed to registered.
      *   3-1. If the new schema is incompatible.
      *     3-1-1. If {@link #pauseSendingToPreservePublishOrderOnSchemaRegFailure} is true pause all following
-     *       publishing to avoid out-of-order issue.
+     *       publishing to avoid out-of-order issue, and retry registration with backoff.
      *     3-1-2. Otherwise, discard the failed message anc continuously publishing the following messages.
      *            Additionally, the following messages may need schema registration also.
-     *   3-2. The new schema registration failed due to other error, retry registering.
+     *   3-2. The new schema registration failed due to other error, retry registering with backoff.
+     * 4. Retry a previously incompatible schema after backoff.
      * Note: Since the current method accesses & modifies {@link #pendingMessages}, you should acquire a lock on
      *       {@link ProducerImpl} before calling method.
      */
@@ -2910,7 +2954,6 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
         Iterator<OpSendMsg> msgIterator = pendingMessages.iterator();
         MessageImpl<?> loopStartAt = latestMsgAttemptedRegisteredSchema;
         OpSendMsg loopEndDueToSchemaRegisterNeeded = null;
-        boolean pausedSendingToPreservePublishOrderOnSchemaRegFailure = false;
         while (true) {
             // Re-check after every iteration that can invoke application callbacks, including the
             // last one: closing the producer may also drain the queue and invalidate this iterator.
@@ -2938,8 +2981,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
             }
             if (op.msg != null) {
                 if (Broken.equals(op.msg.getSchemaState())) {
-                    // "Event 1-1" happens after "Event 3-1-1".
-                    // Maybe user has changed the schema compatibility strategy, will retry to register the new schema.
+                    // Retry a previously incompatible schema after backoff or reconnect.
                     if (pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
                         loopEndDueToSchemaRegisterNeeded = op;
                         break;
@@ -2973,12 +3015,9 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
                                 .attr("details", SchemaUtils.jsonifySchemaInfo(msgSchemaInfo, false))
                                 .log("Publishing paused: message schema"
                                         + " incompatible with target cluster."
-                                        + " To resume publishing: 1) Adjust"
-                                        + " schema compatibility strategy on"
-                                        + " target cluster 2) Unload topic on"
-                                        + " target cluster. Schema details");
+                                        + " Schema registration will be retried"
+                                        + " automatically. Schema details");
                         loopEndDueToSchemaRegisterNeeded = op;
-                        pausedSendingToPreservePublishOrderOnSchemaRegFailure = true;
                         break;
                     }
                     // Event 3-1-2.
@@ -3045,18 +3084,16 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
                 // "Event 1-1" happens after "Event 3-1-1".
                 // After a topic unload, ask the producer retry to register schema, which avoids restart client
                 // after users changed the compatibility strategy to make the schema is compatible.
-                tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded.msg, loopEndDueToSchemaRegisterNeeded.callback,
-                    expectedEpoch);
-            } else if (pausedSendingToPreservePublishOrderOnSchemaRegFailure) {
-                // Nothing to do if the event is "Event 3-1-1", just keep stuck.
+                tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded, expectedEpoch);
+            } else if (failedIncompatibleSchema && pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
+                // The registration failure handler schedules the retry, keeping subsequent messages queued.
                 return;
             } else if (compareAndSetState(State.RegisteringSchema, State.Ready)) {
                 // "Event 2-1" or "Event 3-1-2" or "Event 3-2".
                 // "pendingMessages" has more messages to register new schema.
                 // This operation will not be conflict with another schema registration because both operations are
                 // attempt to acquire the same lock "ProducerImpl.this".
-                tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded.msg, loopEndDueToSchemaRegisterNeeded.callback,
-                        expectedEpoch);
+                tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded, expectedEpoch);
             }
             // Schema registration will trigger a new "recoverProcessOpSendMsgFrom", so return here. If failed to switch
             // state, it means another task will trigger a new "recoverProcessOpSendMsgFrom".
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
index 9fce50f781ad..02be50b81ccf 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pulsar.client.impl;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.Mockito.CALLS_REAL_METHODS;
 import static org.mockito.Mockito.any;
@@ -50,6 +51,7 @@ import io.netty.util.Timer;
 import io.netty.util.concurrent.ScheduledFuture;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -57,6 +59,7 @@ import java.util.Optional;
 import java.util.TreeSet;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.commons.lang3.reflect.FieldUtils;
@@ -74,6 +77,7 @@ import org.apache.pulsar.client.impl.metrics.InstrumentProvider;
 import org.apache.pulsar.client.impl.metrics.LatencyHistogram;
 import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.apache.pulsar.common.protocol.ByteBufPair;
+import org.apache.pulsar.common.protocol.Commands;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 import org.testng.annotations.DataProvider;
@@ -341,6 +345,346 @@ public class ProducerImplTest {
                 .defaultAnswer(CALLS_REAL_METHODS));
     }
 
+    @Test
+    public void testReplicationSchemaRegistrationRetriesWithBackoff() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            CompletableFuture<?> blocked = producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            CompletableFuture<?> following = producer.newMessage().value(new byte[] {1}).sendAsync();
+            for (int i = 0; i < 5; i++) {
+                fixture.rejectSchema();
+                assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+                assertThat(producer.pendingMessages.size()).isEqualTo(2);
+                assertThat(blocked).isNotDone();
+                assertThat(following).isNotDone();
+                assertThat(fixture.writes).hasValue(0);
+                long delay = Math.min(100L << i, 400L);
+                assertThat(fixture.retryDelays.get(i)).isBetween(delay * 95 / 100, delay * 105 / 100);
+                fixture.retries.get(i).run();
+                assertThat(fixture.requests).hasSize(i + 2);
+            }
+            fixture.requests.get(5).complete(new byte[] {1});
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+            assertThat(fixture.writes).hasValue(2);
+            producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+            producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+            assertThat(blocked).isCompleted();
+            assertThat(following).isCompleted();
+            assertThat(producer.pendingMessages.size()).isZero();
+
+            // A different schema conflict starts a fresh backoff sequence.
+            producer.newMessage(Schema.BOOL).value(true).sendAsync();
+            fixture.rejectSchema();
+            assertThat(fixture.retryDelays.get(5)).isBetween(95L, 105L);
+        }
+    }
+
+    @Test
+    public void testSchemaRegistrationRetryStopsAfterProducerClose() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.rejectSchema();
+            assertThat(fixture.retries).hasSize(1);
+            fixture.producer.closeAsync().get(5, TimeUnit.SECONDS);
+            fixture.retries.get(0).run();
+            assertThat(fixture.requests).hasSize(1);
+            assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Closed);
+            assertThat(blocked).isCompletedExceptionally();
+            verify(fixture.retryTasks.get(0)).cancel(false);
+            verify(fixture.cnx.channel(), never()).close();
+        }
+    }
+
+    @DataProvider
+    public Object[][] staleSchemaRetries() {
+        return new Object[][] {{false}, {true}};
+    }
+
+    @Test(dataProvider = "staleSchemaRetries")
+    public void testSchemaRetryCannotInterruptAnotherRegistration(boolean nextMessage) throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            producer.newMessage().value(new byte[] {0}).sendAsync();
+            producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.rejectSchema();
+            Runnable staleRetry = fixture.retries.get(0);
+            // A checksum resend can start a new registration without changing the connection epoch.
+            producer.recoverChecksumError(fixture.cnx, -1);
+            assertThat(fixture.requests).hasSize(2);
+            if (nextMessage) {
+                fixture.requests.get(1).complete(new byte[] {1});
+                producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+                producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+                producer.newMessage(Schema.BOOL).value(true).sendAsync();
+                assertThat(fixture.requests).hasSize(3);
+            }
+            int requests = fixture.requests.size();
+            int writes = fixture.writes.get();
+            staleRetry.run();
+            assertThat(fixture.requests).hasSize(requests);
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+            producer.newMessage().value(new byte[] {2}).sendAsync();
+            assertThat(fixture.writes).hasValue(writes);
+            verify(fixture.retryTasks.get(0)).cancel(false);
+            verify(fixture.cnx.channel(), never()).close();
+        }
+    }
+
+    @Test(dataProvider = "staleSchemaRetries")
+    public void testStaleSchemaResponseCannotInterruptAnotherRegistration(boolean success) throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            producer.newMessage().value(new byte[] {0}).sendAsync();
+            producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            producer.recoverChecksumError(fixture.cnx, -1);
+            assertThat(fixture.requests).hasSize(2);
+            int writes = fixture.writes.get();
+            if (success) {
+                fixture.requests.get(0).complete(new byte[] {1});
+            } else {
+                fixture.requests.get(0).completeExceptionally(
+                        new PulsarClientException.IncompatibleSchemaException("stale rejection"));
+            }
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+            assertThat(fixture.retries).isEmpty();
+            assertThat(fixture.writes).hasValue(writes);
+            fixture.requests.get(1).complete(new byte[] {1});
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+            assertThat(fixture.writes).hasValue(writes + 1);
+        }
+    }
+
+    @Test(dataProvider = "staleSchemaRetries")
+    public void testSchemaResponseAfterCloseIsIgnored(boolean success) throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.producer.closeAsync().get(5, TimeUnit.SECONDS);
+            if (success) {
+                fixture.requests.get(0).complete(new byte[] {1});
+            } else {
+                fixture.requests.get(0).completeExceptionally(new PulsarClientException("late failure"));
+            }
+            assertThat(blocked).isCompletedExceptionally();
+            assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Closed);
+            assertThat(fixture.retries).isEmpty();
+            assertThat(fixture.writes).hasValue(0);
+            verify(fixture.cnx.channel(), never()).close();
+        }
+    }
+
+    @Test
+    public void testSchemaResponseRecoversAfterPendingMessageFails() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            CompletableFuture<?> expired = producer.newMessage(Schema.STRING).value("expired").sendAsync();
+            producer.failPendingMessages(null, new PulsarClientException.TimeoutException("send timed out"));
+            producer.newMessage().value(new byte[] {1}).sendAsync();
+            fixture.requests.get(0).complete(new byte[] {1});
+            assertThat(expired).isCompletedExceptionally();
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+            assertThat(fixture.writes).hasValue(1);
+            assertThat(fixture.requests).hasSize(1);
+        }
+    }
+
+    @Test
+    public void testSchemaRetryDoesNotRequireBlockedMessageAtQueueHead() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            producer.newMessage().value(new byte[] {0}).sendAsync();
+            producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.rejectSchema();
+            fixture.retries.get(0).run();
+            assertThat(fixture.requests).hasSize(2);
+            fixture.requests.get(1).complete(new byte[] {1});
+            assertThat(fixture.writes).hasValue(2);
+            assertThat(producer.pendingMessages.size()).isEqualTo(2);
+        }
+    }
+
+    @DataProvider
+    public Object[][] transientSchemaErrors() {
+        return new Object[][] {
+                {false, new PulsarClientException("registration failed")},
+                {true, new PulsarClientException("registration failed")},
+                {false, new PulsarClientException.NotSupportedException("unsupported")},
+                {true, new PulsarClientException.NotSupportedException("unsupported")}
+        };
+    }
+
+    @Test(dataProvider = "transientSchemaErrors")
+    public void testSchemaRegistrationErrorsUseBackoff(boolean replication, PulsarClientException error)
+            throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(replication)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            CompletableFuture<?> blocked = producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            producer.newMessage().value(new byte[] {1}).sendAsync();
+            for (int i = 0; i < 5; i++) {
+                fixture.requests.get(i).completeExceptionally(error);
+                assertThat(fixture.requests).hasSize(i + 1);
+                assertThat(fixture.retries).hasSize(i + 1);
+                assertThat(blocked).isNotDone();
+                assertThat(fixture.writes).hasValue(0);
+                long delay = Math.min(100L << i, 400L);
+                assertThat(fixture.retryDelays.get(i)).isBetween(delay * 95 / 100, delay * 105 / 100);
+                fixture.retries.get(i).run();
+            }
+            fixture.requests.get(5).complete(new byte[] {1});
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+            assertThat(fixture.writes).hasValue(2);
+            producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+            producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+            assertThat(blocked).isCompleted();
+        }
+    }
+
+    @Test
+    public void testUnsupportedSchemaProtocolUsesBackoff() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+            when(fixture.cnx.getRemoteEndpointProtocolVersion()).thenReturn(0);
+            CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            for (int i = 0; i < 3; i++) {
+                assertThat(fixture.requests).isEmpty();
+                assertThat(fixture.retries).hasSize(i + 1);
+                assertThat(blocked).isNotDone();
+                fixture.retries.get(i).run();
+            }
+            when(fixture.cnx.getRemoteEndpointProtocolVersion()).thenReturn(Commands.getCurrentProtocolVersion());
+            fixture.retries.get(3).run();
+            assertThat(fixture.requests).hasSize(1);
+            fixture.requests.get(0).complete(new byte[] {1});
+            assertThat(fixture.writes).hasValue(1);
+        }
+    }
+
+    @Test
+    public void testSchemaRetryRecoversAfterPendingMessageFails() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+            ProducerImpl<byte[]> producer = fixture.producer;
+            CompletableFuture<?> expired = producer.newMessage(Schema.STRING).value("expired").sendAsync();
+            fixture.requests.get(0).completeExceptionally(new PulsarClientException("registration failed"));
+            producer.failPendingMessages(null, new PulsarClientException.TimeoutException("send timed out"));
+            CompletableFuture<?> following = producer.newMessage().value(new byte[] {1}).sendAsync();
+            fixture.retries.get(0).run();
+            assertThat(expired).isCompletedExceptionally();
+            assertThat(following).isNotDone();
+            assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+            assertThat(fixture.writes).hasValue(1);
+        }
+    }
+
+    @DataProvider
+    public Object[][] schemaRetryConnectionChanges() {
+        return new Object[][] {{false}, {true}};
+    }
+
+    @Test(dataProvider = "schemaRetryConnectionChanges")
+    public void testSchemaRegistrationRetryIgnoresStaleConnection(boolean sameConnection) throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+            fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.rejectSchema();
+            assertThat(fixture.retries).hasSize(1);
+            // Re-registering on a shared TCP connection must invalidate the old epoch too.
+            fixture.producer.getConnectionHandler().switchClientCnx(sameConnection
+                    ? fixture.cnx : mock(ClientCnx.class));
+            fixture.retries.get(0).run();
+            assertThat(fixture.requests).hasSize(1);
+            assertThat(fixture.retries).hasSize(1);
+            assertThat(fixture.producer.pendingMessages.size()).isEqualTo(1);
+            verify(fixture.cnx.channel(), never()).close();
+        }
+    }
+
+    @Test
+    public void testOrdinaryProducerStillFailsIncompatibleSchema() throws Exception {
+        try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+            CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+            fixture.rejectSchema();
+            assertThatThrownBy(blocked::join)
+                    .hasCauseInstanceOf(PulsarClientException.IncompatibleSchemaException.class);
+            assertThat(fixture.retries).isEmpty();
+            assertThat(fixture.producer.pendingMessages.size()).isZero();
+            assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Ready);
+        }
+    }
+
+    private static final class SchemaRetryFixture implements AutoCloseable {
+        private final ClientCnx cnx = mock(ClientCnx.class);
+        private final ProducerImpl<byte[]> producer;
+        private final List<CompletableFuture<byte[]>> requests = new ArrayList<>();
+        private final List<Runnable> retries = new ArrayList<>();
+        private final List<Long> retryDelays = new ArrayList<>();
+        private final List<ScheduledFuture<?>> retryTasks = new ArrayList<>();
+        private final AtomicInteger writes = new AtomicInteger();
+
+        private SchemaRetryFixture(boolean replicationProducer) {
+            PulsarClientImpl client = mockedPulsarClient();
+            when(client.getMemoryLimitController()).thenReturn(new MemoryLimitController(0));
+            when(client.getClientClock()).thenReturn(Clock.systemUTC());
+            client.getConfiguration().setInitialBackoffIntervalNanos(TimeUnit.MILLISECONDS.toNanos(100));
+            client.getConfiguration().setMaxBackoffIntervalNanos(TimeUnit.MILLISECONDS.toNanos(400));
+            ProducerConfigurationData conf = new ProducerConfigurationData();
+            conf.setBatchingEnabled(false);
+            conf.setSendTimeoutMs(0);
+            conf.setProducerName("schema-retry");
+            conf.setReplProducer(replicationProducer);
+            producer = constructProducer(client, conf);
+            EventLoop eventLoop = mock(EventLoop.class);
+            doAnswer(invocation -> {
+                invocation.<Runnable>getArgument(0).run();
+                return null;
+            }).when(eventLoop).execute(any(Runnable.class));
+            doAnswer(invocation -> {
+                retries.add(invocation.getArgument(0));
+                retryDelays.add(invocation.getArgument(1));
+                ScheduledFuture<?> task = mock(ScheduledFuture.class);
+                retryTasks.add(task);
+                return task;
+            }).when(eventLoop).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+            Channel channel = mock(Channel.class);
+            when(channel.eventLoop()).thenReturn(eventLoop);
+            ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+            when(ctx.channel()).thenReturn(channel);
+            when(cnx.ctx()).thenReturn(ctx);
+            when(cnx.channel()).thenReturn(channel);
+            when(cnx.getRemoteEndpointProtocolVersion()).thenReturn(Commands.getCurrentProtocolVersion());
+            doAnswer(invocation -> {
+                ReferenceCountUtil.release(invocation.getArgument(0));
+                writes.incrementAndGet();
+                return null;
+            }).when(ctx).write(any(), any());
+            doAnswer(invocation -> {
+                ReferenceCountUtil.release(invocation.getArgument(0));
+                writes.incrementAndGet();
+                return null;
+            }).when(ctx).writeAndFlush(any(), any());
+            doAnswer(invocation -> {
+                ReferenceCountUtil.release(invocation.getArgument(0));
+                CompletableFuture<byte[]> response = new CompletableFuture<>();
+                requests.add(response);
+                return response;
+            }).when(cnx).sendGetOrCreateSchema(any(), anyLong());
+            doAnswer(invocation -> {
+                ReferenceCountUtil.release(invocation.getArgument(0));
+                return CompletableFuture.completedFuture(null);
+            }).when(cnx).sendRequestWithId(any(), anyLong());
+            producer.getConnectionHandler().switchClientCnx(cnx);
+            producer.setState(HandlerState.State.Ready);
+        }
+
+        private void rejectSchema() {
+            requests.get(requests.size() - 1).completeExceptionally(
+                    new PulsarClientException.IncompatibleSchemaException("incompatible key schema"));
+        }
+
+        @Override
+        public void close() {
+            // Restore the mocked connection so close can clean up even after a stale-epoch test.
+            producer.getConnectionHandler().setClientCnx(cnx);
+            producer.closeAsync().join();
+        }
+    }
+
     /**
      * When encryption fails in any way, the partially built encrypted buffer must be released instead of
      * leaking; the source payload stays with the caller (the batch container decides its fate).