You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/21 10:19:47 UTC
(pulsar) branch master updated: [fix][broker] Retry incompatible replication schemas without topic unload (#27026)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new f530dfd030bd [fix][broker] Retry incompatible replication schemas without topic unload (#27026)
f530dfd030bd is described below
commit f530dfd030bdb7464d94eb70c33f6c9a5447e5e2
Author: Penghui Li <pe...@apache.org>
AuthorDate: Wed Oct 21 03:19:40 2026 -0700
[fix][broker] Retry incompatible replication schemas without topic unload (#27026)
---
.../broker/service/OneWayReplicatorTest.java | 101 ++++++
.../apache/pulsar/client/impl/ProducerImpl.java | 149 +++++----
.../pulsar/client/impl/ProducerImplTest.java | 344 +++++++++++++++++++++
3 files changed, 538 insertions(+), 56 deletions(-)
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
index ac20c2eb9dde..dadedc0b932a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
@@ -86,6 +86,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest;
import org.apache.commons.collections4.CollectionUtils;
+import org.apache.logging.log4j.Level;
import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.resources.ClusterResources;
@@ -110,6 +111,9 @@ import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.client.api.schema.GenericRecord;
+import org.apache.pulsar.client.api.schema.GenericSchema;
+import org.apache.pulsar.client.api.schema.RecordSchemaBuilder;
+import org.apache.pulsar.client.api.schema.SchemaBuilder;
import org.apache.pulsar.client.impl.ClientBuilderImpl;
import org.apache.pulsar.client.impl.ClientCnx;
import org.apache.pulsar.client.impl.ClientImplInternalSetter;
@@ -136,11 +140,13 @@ import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.common.policies.data.TopicStats;
import org.apache.pulsar.common.policies.data.TopicType;
import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
+import org.apache.pulsar.common.schema.KeyValue;
import org.apache.pulsar.common.schema.LongSchemaVersion;
import org.apache.pulsar.common.schema.SchemaInfo;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.impl.DualMetadataStore;
+import org.apache.pulsar.utils.TestLogAppender;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
import org.apache.pulsar.zookeeper.ZookeeperServerTest;
import org.awaitility.Awaitility;
@@ -2133,6 +2139,101 @@ public class OneWayReplicatorTest extends OneWayReplicatorTestBase {
};
}
+ @DataProvider
+ public Object[][] schemaRecovery() {
+ return new Object[][] {{false, false}, {false, true}, {true, false}, {true, true}};
+ }
+
+ @Test(dataProvider = "schemaRecovery", timeOut = 120_000)
+ public void testReplicationResumesAfterSchemaConflictResolved(boolean deduplication, boolean deleteSchema)
+ throws Exception {
+ String topicName = BrokerTestUtil.newUniqueName("persistent://"
+ + sourceClusterAlwaysSchemaCompatibleNamespace + "/schema-retry-");
+ RecordSchemaBuilder oldKeyBuilder = SchemaBuilder.record("Key");
+ oldKeyBuilder.field("id").type(SchemaType.INT64);
+ RecordSchemaBuilder newKeyBuilder = SchemaBuilder.record("Key");
+ newKeyBuilder.field("game_id").type(SchemaType.INT64);
+ GenericSchema<GenericRecord> oldKey = Schema.generic(oldKeyBuilder.build(SchemaType.AVRO));
+ GenericSchema<GenericRecord> newKey = Schema.generic(newKeyBuilder.build(SchemaType.AVRO));
+ Schema<KeyValue<GenericRecord, String>> schema = Schema.KeyValue(newKey, Schema.STRING);
+
+ admin2.topics().createNonPartitionedTopic(topicName);
+ admin2.topicPolicies().setSchemaCompatibilityStrategy(topicName, SchemaCompatibilityStrategy.FULL);
+ admin2.topicPolicies().setDeduplicationStatus(topicName, deduplication);
+ admin2.schemas().createSchema(topicName, Schema.KeyValue(oldKey, Schema.STRING).getSchemaInfo());
+ try (Consumer<byte[]> consumer = client2.newConsumer(Schema.BYTES).topic(topicName)
+ .subscriptionName("schema-retry").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+ .subscribe();
+ Producer<byte[]> producer = client1.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName)
+ .enableBatching(false).create();
+ TestLogAppender logs = TestLogAppender.create(ProducerImpl.class)) {
+ waitReplicatorStarted(topicName);
+ PersistentTopic destination = (PersistentTopic) broker2.getTopic(topicName, false).join().orElseThrow();
+ Awaitility.await().untilAsserted(() -> {
+ assertThat(destination.getSchemaCompatibilityStrategy()).isEqualTo(SchemaCompatibilityStrategy.FULL);
+ assertThat(destination.getMessageDeduplication().isEnabled()).isEqualTo(deduplication);
+ });
+ PersistentTopic source = (PersistentTopic) broker1.getTopic(topicName, false).join().orElseThrow();
+ PersistentReplicator replicator = (PersistentReplicator) source.getReplicators().get(cluster2);
+ ProducerImpl<?> replicationProducer = replicator.getProducer();
+ ClientCnx connection = replicationProducer.getClientCnx();
+ long producerEpoch = replicationProducer.getConnectionHandler().getEpoch();
+ List<byte[]> expected = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ KeyValue<GenericRecord, String> value = new KeyValue<>(
+ newKey.newRecordBuilder().set("game_id", (long) i).build(), "game-" + i);
+ producer.newMessage(schema).value(value).send();
+ expected.add(schema.encode(value));
+ // A schema-free message must not overtake the incompatible message.
+ byte[] following = ("after-" + i).getBytes(StandardCharsets.UTF_8);
+ producer.newMessage(Schema.BYTES).value(following).send();
+ expected.add(following);
+ }
+ Awaitility.await().untilAsserted(() -> assertThat(logs.getEvents()).anySatisfy(event ->
+ assertThat(event.getMessage().getFormattedMessage()).contains("Publishing paused")));
+ assertThat(replicator.getCursor().getNumberOfEntriesInBacklog(false)).isEqualTo(expected.size());
+ assertThat(consumer.receive(1, TimeUnit.SECONDS)).isNull();
+ assertThat(logs.getEvents().stream().filter(event -> event.getLevel() == Level.WARN
+ && event.getMessage().getFormattedMessage().contains("GetOrCreateSchema error")).count())
+ .isEqualTo(1);
+ assertThat(logs.getEvents()).noneSatisfy(event -> {
+ assertThat(event.getLevel()).isEqualTo(Level.INFO);
+ assertThat(event.getMessage().getFormattedMessage()).contains("GetOrCreateSchema request");
+ });
+
+ if (deleteSchema) {
+ admin2.schemas().deleteSchema(topicName);
+ } else {
+ admin2.topicPolicies().setSchemaCompatibilityStrategy(topicName,
+ SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE);
+ }
+ Awaitility.await().atMost(Duration.ofSeconds(30)).untilAsserted(() ->
+ assertThat(replicator.getCursor().getNumberOfEntriesInBacklog(false)).isZero());
+ assertThat(admin2.schemas().getSchemaInfo(topicName).getSchema())
+ .isEqualTo(schema.getSchemaInfo().getSchema());
+ assertThat(replicator.getProducer()).isSameAs(replicationProducer);
+ assertThat(replicationProducer.getClientCnx()).isSameAs(connection);
+ assertThat(replicationProducer.getConnectionHandler().getEpoch()).isEqualTo(producerEpoch);
+ assertThat(broker1.getTopic(topicName, false).join().orElseThrow()).isSameAs(source);
+ assertThat(broker2.getTopic(topicName, false).join().orElseThrow()).isSameAs(destination);
+ for (byte[] payload : expected) {
+ Message<byte[]> received = consumer.receive(5, TimeUnit.SECONDS);
+ assertThat(received).isNotNull();
+ assertThat(received.getData()).isEqualTo(payload);
+ consumer.acknowledge(received);
+ }
+ assertThat(consumer.receive(1, TimeUnit.SECONDS)).isNull();
+ producer.newMessage(schema).value(new KeyValue<>(
+ newKey.newRecordBuilder().set("game_id", 100L).build(), "live")).send();
+ assertThat(consumer.receive(5, TimeUnit.SECONDS)).isNotNull();
+ } finally {
+ admin1.topics().setReplicationClusters(topicName, Collections.singletonList(cluster1));
+ waitReplicatorStopped(topicName, false);
+ admin1.topics().delete(topicName, true);
+ admin2.topics().delete(topicName, true);
+ }
+ }
+
@Test(dataProvider = "enableDeduplication")
public void testIncompatibleMultiVersionSchema(boolean enableDeduplication) throws Exception {
final String topicName = BrokerTestUtil.newUniqueName("persistent://"
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
index 84e2042ea840..3bbdee956b47 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java
@@ -201,6 +201,11 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
private final Counter producersOpenedCounter;
private final Counter producersClosedCounter;
private final boolean pauseSendingToPreservePublishOrderOnSchemaRegFailure;
+ private final Backoff schemaRegistrationBackoff;
+ // Guarded by the producer lock, including schema callbacks and retry tasks.
+ private long schemaRegistrationGeneration;
+ private ScheduledFuture<?> schemaRegistrationRetryTask;
+ private boolean schemaRegistrationFailed;
// This variable can be exposed as a metrics in the future, a PIP is needed.
private final AtomicInteger pendingQueueFullCounter;
@@ -218,6 +223,10 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
// Otherwise, it may cause an out-of-order issue, and it may lead to a messages lost issue if users enabled
// deduplication on the remote side.
this.pauseSendingToPreservePublishOrderOnSchemaRegFailure = conf.isReplProducer();
+ this.schemaRegistrationBackoff = Backoff.builder()
+ .initialDelay(Duration.ofNanos(client.getConfiguration().getInitialBackoffIntervalNanos()))
+ .maxBackoff(Duration.ofNanos(client.getConfiguration().getMaxBackoffIntervalNanos()))
+ .build();
if (conf.getMaxPendingMessages() > 0) {
// With admission done upstream the permit is never taken but would still be released.
checkArgument(!conf.isMemoryLimitAdmittedUpstream(),
@@ -982,53 +991,85 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
return true;
}
- private void tryRegisterSchema(ClientCnx cnx, final MessageImpl<?> msg, SendCallback callback,
- long expectedCnxEpoch) {
+ private void tryRegisterSchema(ClientCnx cnx, OpSendMsg op, long expectedCnxEpoch) {
if (!changeToRegisteringSchemaState()) {
return;
}
+ long generation = ++schemaRegistrationGeneration;
+ cancelSchemaRegistrationRetry();
+ MessageImpl<?> msg = op.msg;
+ long sequenceId = op.sequenceId;
SchemaInfo schemaInfo = msg.hasReplicateFrom() ? msg.getSchemaInfoForReplicator() : msg.getSchemaInfo();
schemaInfo = Optional.ofNullable(schemaInfo)
.filter(si -> si.getType().getValue() > 0)
.orElse(Schema.BYTES.getSchemaInfo());
- getOrCreateSchemaAsync(cnx, schemaInfo).handle((v, ex) -> {
- if (ex != null) {
- Throwable t = FutureUtil.unwrapCompletionException(ex);
- log.warn()
- .exception(t)
- .log("GetOrCreateSchema error");
- if (t instanceof PulsarClientException.IncompatibleSchemaException) {
- // Only the first time of failed schema registration will trigger a "recoverProcessOpSendMsgFrom".
- if (!Broken.equals(msg.getSchemaState())) {
- cnx.ctx().channel().eventLoop().execute(() -> {
- synchronized (ProducerImpl.this) {
+ getOrCreateSchemaAsync(cnx, schemaInfo).whenComplete((v, ex) ->
+ cnx.ctx().channel().eventLoop().execute(() -> {
+ synchronized (ProducerImpl.this) {
+ if (generation != schemaRegistrationGeneration || cnx() != cnx
+ || expectedCnxEpoch != connectionHandler.getEpoch()
+ || getState() != State.RegisteringSchema) {
+ return;
+ }
+ // A send timeout can dispose and recycle the operation while registration is outstanding.
+ if (op.msg != msg || op.sequenceId != sequenceId) {
+ recoverProcessOpSendMsgFrom(cnx, null, false, expectedCnxEpoch);
+ return;
+ }
+ if (ex == null) {
+ log.debug("GetOrCreateSchema succeed");
+ // An empty version means there is no schema and must not be set on message metadata.
+ if (v.length != 0) {
+ schemaCache.putIfAbsent(msg.getSchemaHash(), v);
+ msg.getMessageBuilder().setSchemaVersion(v);
+ } else {
+ schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes());
+ }
+ msg.setSchemaState(MessageImpl.SchemaState.Ready);
+ schemaRegistrationBackoff.reset();
+ schemaRegistrationFailed = false;
+ recoverProcessOpSendMsgFrom(cnx, msg, false, expectedCnxEpoch);
+ return;
+ }
+ Throwable t = FutureUtil.unwrapCompletionException(ex);
+ if (schemaRegistrationFailed) {
+ log.debug().exceptionMessage(t).log("GetOrCreateSchema retry failed");
+ } else {
+ log.warn().exception(t).log("GetOrCreateSchema error");
+ }
+ schemaRegistrationFailed = true;
+ if (t instanceof IncompatibleSchemaException) {
+ if (!pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
+ schemaRegistrationBackoff.reset();
+ schemaRegistrationFailed = false;
recoverProcessOpSendMsgFrom(cnx, msg, true, expectedCnxEpoch);
+ return;
}
- });
+ recoverProcessOpSendMsgFrom(cnx, msg, true, expectedCnxEpoch);
+ }
+ schemaRegistrationRetryTask = cnx.ctx().channel().eventLoop().schedule(() -> {
+ synchronized (ProducerImpl.this) {
+ if (generation != schemaRegistrationGeneration) {
+ return;
+ }
+ schemaRegistrationRetryTask = null;
+ if (cnx() == cnx && expectedCnxEpoch == connectionHandler.getEpoch()
+ && getState() == State.RegisteringSchema) {
+ MessageImpl<?> retryFrom =
+ op.msg == msg && op.sequenceId == sequenceId ? msg : null;
+ recoverProcessOpSendMsgFrom(cnx, retryFrom, false, expectedCnxEpoch);
+ }
+ }
+ }, schemaRegistrationBackoff.next().toMillis(), TimeUnit.MILLISECONDS);
}
- return null;
- }
- } else {
- log.info("GetOrCreateSchema succeed");
- // In broker, if schema version is an empty byte array, it means the topic doesn't have schema.
- // In this case, we cache the schema version to `SchemaVersion.Empty.bytes()`.
- // When we need to set the schema version of the message metadata,
- // we should check if the cached schema version is `SchemaVersion.Empty.bytes()`
- if (v.length != 0) {
- schemaCache.putIfAbsent(msg.getSchemaHash(), v);
- msg.getMessageBuilder().setSchemaVersion(v);
- } else {
- schemaCache.putIfAbsent(msg.getSchemaHash(), SchemaVersion.Empty.bytes());
- }
- msg.setSchemaState(MessageImpl.SchemaState.Ready);
- }
- cnx.ctx().channel().eventLoop().execute(() -> {
- synchronized (ProducerImpl.this) {
- recoverProcessOpSendMsgFrom(cnx, msg, false, expectedCnxEpoch);
- }
- });
- return null;
- });
+ }));
+ }
+
+ private void cancelSchemaRegistrationRetry() {
+ if (schemaRegistrationRetryTask != null) {
+ schemaRegistrationRetryTask.cancel(false);
+ schemaRegistrationRetryTask = null;
+ }
}
private CompletableFuture<byte[]> getOrCreateSchemaAsync(ClientCnx cnx, SchemaInfo schemaInfo) {
@@ -1041,7 +1082,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
}
long requestId = client.newRequestId();
ByteBuf request = Commands.newGetOrCreateSchema(requestId, topic, producerName, schemaInfo);
- log.info("GetOrCreateSchema request");
+ log.debug("GetOrCreateSchema request");
return cnx.sendGetOrCreateSchema(request, requestId);
}
@@ -2401,7 +2442,9 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
return true;
}
- private void closeProducerTasks() {
+ private synchronized void closeProducerTasks() {
+ schemaRegistrationGeneration++;
+ cancelSchemaRegistrationRetry();
Timeout timeout = sendTimeout;
if (timeout != null) {
timeout.cancel();
@@ -2811,7 +2854,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
final ClientCnx cnx = getCnxIfReady();
if (cnx != null) {
if (op.msg != null && op.msg.getSchemaState() == None) {
- tryRegisterSchema(cnx, op.msg, op.callback, this.connectionHandler.getEpoch());
+ tryRegisterSchema(cnx, op, this.connectionHandler.getEpoch());
return;
}
// If we do have a connection, the message is sent immediately, otherwise we'll try again once a new
@@ -2886,10 +2929,11 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
* 3. If using multiple version producer, the new schema was failed to registered.
* 3-1. If the new schema is incompatible.
* 3-1-1. If {@link #pauseSendingToPreservePublishOrderOnSchemaRegFailure} is true pause all following
- * publishing to avoid out-of-order issue.
+ * publishing to avoid out-of-order issue, and retry registration with backoff.
* 3-1-2. Otherwise, discard the failed message anc continuously publishing the following messages.
* Additionally, the following messages may need schema registration also.
- * 3-2. The new schema registration failed due to other error, retry registering.
+ * 3-2. The new schema registration failed due to other error, retry registering with backoff.
+ * 4. Retry a previously incompatible schema after backoff.
* Note: Since the current method accesses & modifies {@link #pendingMessages}, you should acquire a lock on
* {@link ProducerImpl} before calling method.
*/
@@ -2910,7 +2954,6 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
Iterator<OpSendMsg> msgIterator = pendingMessages.iterator();
MessageImpl<?> loopStartAt = latestMsgAttemptedRegisteredSchema;
OpSendMsg loopEndDueToSchemaRegisterNeeded = null;
- boolean pausedSendingToPreservePublishOrderOnSchemaRegFailure = false;
while (true) {
// Re-check after every iteration that can invoke application callbacks, including the
// last one: closing the producer may also drain the queue and invalidate this iterator.
@@ -2938,8 +2981,7 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
}
if (op.msg != null) {
if (Broken.equals(op.msg.getSchemaState())) {
- // "Event 1-1" happens after "Event 3-1-1".
- // Maybe user has changed the schema compatibility strategy, will retry to register the new schema.
+ // Retry a previously incompatible schema after backoff or reconnect.
if (pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
loopEndDueToSchemaRegisterNeeded = op;
break;
@@ -2973,12 +3015,9 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
.attr("details", SchemaUtils.jsonifySchemaInfo(msgSchemaInfo, false))
.log("Publishing paused: message schema"
+ " incompatible with target cluster."
- + " To resume publishing: 1) Adjust"
- + " schema compatibility strategy on"
- + " target cluster 2) Unload topic on"
- + " target cluster. Schema details");
+ + " Schema registration will be retried"
+ + " automatically. Schema details");
loopEndDueToSchemaRegisterNeeded = op;
- pausedSendingToPreservePublishOrderOnSchemaRegFailure = true;
break;
}
// Event 3-1-2.
@@ -3045,18 +3084,16 @@ public class ProducerImpl<T> extends ProducerBase<T> implements TimerTask, Conne
// "Event 1-1" happens after "Event 3-1-1".
// After a topic unload, ask the producer retry to register schema, which avoids restart client
// after users changed the compatibility strategy to make the schema is compatible.
- tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded.msg, loopEndDueToSchemaRegisterNeeded.callback,
- expectedEpoch);
- } else if (pausedSendingToPreservePublishOrderOnSchemaRegFailure) {
- // Nothing to do if the event is "Event 3-1-1", just keep stuck.
+ tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded, expectedEpoch);
+ } else if (failedIncompatibleSchema && pauseSendingToPreservePublishOrderOnSchemaRegFailure) {
+ // The registration failure handler schedules the retry, keeping subsequent messages queued.
return;
} else if (compareAndSetState(State.RegisteringSchema, State.Ready)) {
// "Event 2-1" or "Event 3-1-2" or "Event 3-2".
// "pendingMessages" has more messages to register new schema.
// This operation will not be conflict with another schema registration because both operations are
// attempt to acquire the same lock "ProducerImpl.this".
- tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded.msg, loopEndDueToSchemaRegisterNeeded.callback,
- expectedEpoch);
+ tryRegisterSchema(cnx, loopEndDueToSchemaRegisterNeeded, expectedEpoch);
}
// Schema registration will trigger a new "recoverProcessOpSendMsgFrom", so return here. If failed to switch
// state, it means another task will trigger a new "recoverProcessOpSendMsgFrom".
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
index 9fce50f781ad..02be50b81ccf 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerImplTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.client.impl;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.any;
@@ -50,6 +51,7 @@ import io.netty.util.Timer;
import io.netty.util.concurrent.ScheduledFuture;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -57,6 +59,7 @@ import java.util.Optional;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.reflect.FieldUtils;
@@ -74,6 +77,7 @@ import org.apache.pulsar.client.impl.metrics.InstrumentProvider;
import org.apache.pulsar.client.impl.metrics.LatencyHistogram;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.protocol.ByteBufPair;
+import org.apache.pulsar.common.protocol.Commands;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.testng.annotations.DataProvider;
@@ -341,6 +345,346 @@ public class ProducerImplTest {
.defaultAnswer(CALLS_REAL_METHODS));
}
+ @Test
+ public void testReplicationSchemaRegistrationRetriesWithBackoff() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ CompletableFuture<?> blocked = producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ CompletableFuture<?> following = producer.newMessage().value(new byte[] {1}).sendAsync();
+ for (int i = 0; i < 5; i++) {
+ fixture.rejectSchema();
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+ assertThat(producer.pendingMessages.size()).isEqualTo(2);
+ assertThat(blocked).isNotDone();
+ assertThat(following).isNotDone();
+ assertThat(fixture.writes).hasValue(0);
+ long delay = Math.min(100L << i, 400L);
+ assertThat(fixture.retryDelays.get(i)).isBetween(delay * 95 / 100, delay * 105 / 100);
+ fixture.retries.get(i).run();
+ assertThat(fixture.requests).hasSize(i + 2);
+ }
+ fixture.requests.get(5).complete(new byte[] {1});
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+ assertThat(fixture.writes).hasValue(2);
+ producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+ producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+ assertThat(blocked).isCompleted();
+ assertThat(following).isCompleted();
+ assertThat(producer.pendingMessages.size()).isZero();
+
+ // A different schema conflict starts a fresh backoff sequence.
+ producer.newMessage(Schema.BOOL).value(true).sendAsync();
+ fixture.rejectSchema();
+ assertThat(fixture.retryDelays.get(5)).isBetween(95L, 105L);
+ }
+ }
+
+ @Test
+ public void testSchemaRegistrationRetryStopsAfterProducerClose() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.rejectSchema();
+ assertThat(fixture.retries).hasSize(1);
+ fixture.producer.closeAsync().get(5, TimeUnit.SECONDS);
+ fixture.retries.get(0).run();
+ assertThat(fixture.requests).hasSize(1);
+ assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Closed);
+ assertThat(blocked).isCompletedExceptionally();
+ verify(fixture.retryTasks.get(0)).cancel(false);
+ verify(fixture.cnx.channel(), never()).close();
+ }
+ }
+
+ @DataProvider
+ public Object[][] staleSchemaRetries() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @Test(dataProvider = "staleSchemaRetries")
+ public void testSchemaRetryCannotInterruptAnotherRegistration(boolean nextMessage) throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ producer.newMessage().value(new byte[] {0}).sendAsync();
+ producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.rejectSchema();
+ Runnable staleRetry = fixture.retries.get(0);
+ // A checksum resend can start a new registration without changing the connection epoch.
+ producer.recoverChecksumError(fixture.cnx, -1);
+ assertThat(fixture.requests).hasSize(2);
+ if (nextMessage) {
+ fixture.requests.get(1).complete(new byte[] {1});
+ producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+ producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+ producer.newMessage(Schema.BOOL).value(true).sendAsync();
+ assertThat(fixture.requests).hasSize(3);
+ }
+ int requests = fixture.requests.size();
+ int writes = fixture.writes.get();
+ staleRetry.run();
+ assertThat(fixture.requests).hasSize(requests);
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+ producer.newMessage().value(new byte[] {2}).sendAsync();
+ assertThat(fixture.writes).hasValue(writes);
+ verify(fixture.retryTasks.get(0)).cancel(false);
+ verify(fixture.cnx.channel(), never()).close();
+ }
+ }
+
+ @Test(dataProvider = "staleSchemaRetries")
+ public void testStaleSchemaResponseCannotInterruptAnotherRegistration(boolean success) throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ producer.newMessage().value(new byte[] {0}).sendAsync();
+ producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ producer.recoverChecksumError(fixture.cnx, -1);
+ assertThat(fixture.requests).hasSize(2);
+ int writes = fixture.writes.get();
+ if (success) {
+ fixture.requests.get(0).complete(new byte[] {1});
+ } else {
+ fixture.requests.get(0).completeExceptionally(
+ new PulsarClientException.IncompatibleSchemaException("stale rejection"));
+ }
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.RegisteringSchema);
+ assertThat(fixture.retries).isEmpty();
+ assertThat(fixture.writes).hasValue(writes);
+ fixture.requests.get(1).complete(new byte[] {1});
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+ assertThat(fixture.writes).hasValue(writes + 1);
+ }
+ }
+
+ @Test(dataProvider = "staleSchemaRetries")
+ public void testSchemaResponseAfterCloseIsIgnored(boolean success) throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.producer.closeAsync().get(5, TimeUnit.SECONDS);
+ if (success) {
+ fixture.requests.get(0).complete(new byte[] {1});
+ } else {
+ fixture.requests.get(0).completeExceptionally(new PulsarClientException("late failure"));
+ }
+ assertThat(blocked).isCompletedExceptionally();
+ assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Closed);
+ assertThat(fixture.retries).isEmpty();
+ assertThat(fixture.writes).hasValue(0);
+ verify(fixture.cnx.channel(), never()).close();
+ }
+ }
+
+ @Test
+ public void testSchemaResponseRecoversAfterPendingMessageFails() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ CompletableFuture<?> expired = producer.newMessage(Schema.STRING).value("expired").sendAsync();
+ producer.failPendingMessages(null, new PulsarClientException.TimeoutException("send timed out"));
+ producer.newMessage().value(new byte[] {1}).sendAsync();
+ fixture.requests.get(0).complete(new byte[] {1});
+ assertThat(expired).isCompletedExceptionally();
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+ assertThat(fixture.writes).hasValue(1);
+ assertThat(fixture.requests).hasSize(1);
+ }
+ }
+
+ @Test
+ public void testSchemaRetryDoesNotRequireBlockedMessageAtQueueHead() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ producer.newMessage().value(new byte[] {0}).sendAsync();
+ producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.rejectSchema();
+ fixture.retries.get(0).run();
+ assertThat(fixture.requests).hasSize(2);
+ fixture.requests.get(1).complete(new byte[] {1});
+ assertThat(fixture.writes).hasValue(2);
+ assertThat(producer.pendingMessages.size()).isEqualTo(2);
+ }
+ }
+
+ @DataProvider
+ public Object[][] transientSchemaErrors() {
+ return new Object[][] {
+ {false, new PulsarClientException("registration failed")},
+ {true, new PulsarClientException("registration failed")},
+ {false, new PulsarClientException.NotSupportedException("unsupported")},
+ {true, new PulsarClientException.NotSupportedException("unsupported")}
+ };
+ }
+
+ @Test(dataProvider = "transientSchemaErrors")
+ public void testSchemaRegistrationErrorsUseBackoff(boolean replication, PulsarClientException error)
+ throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(replication)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ CompletableFuture<?> blocked = producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ producer.newMessage().value(new byte[] {1}).sendAsync();
+ for (int i = 0; i < 5; i++) {
+ fixture.requests.get(i).completeExceptionally(error);
+ assertThat(fixture.requests).hasSize(i + 1);
+ assertThat(fixture.retries).hasSize(i + 1);
+ assertThat(blocked).isNotDone();
+ assertThat(fixture.writes).hasValue(0);
+ long delay = Math.min(100L << i, 400L);
+ assertThat(fixture.retryDelays.get(i)).isBetween(delay * 95 / 100, delay * 105 / 100);
+ fixture.retries.get(i).run();
+ }
+ fixture.requests.get(5).complete(new byte[] {1});
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+ assertThat(fixture.writes).hasValue(2);
+ producer.ackReceived(fixture.cnx, 0, 0, 1, 0);
+ producer.ackReceived(fixture.cnx, 1, 1, 1, 1);
+ assertThat(blocked).isCompleted();
+ }
+ }
+
+ @Test
+ public void testUnsupportedSchemaProtocolUsesBackoff() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+ when(fixture.cnx.getRemoteEndpointProtocolVersion()).thenReturn(0);
+ CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ for (int i = 0; i < 3; i++) {
+ assertThat(fixture.requests).isEmpty();
+ assertThat(fixture.retries).hasSize(i + 1);
+ assertThat(blocked).isNotDone();
+ fixture.retries.get(i).run();
+ }
+ when(fixture.cnx.getRemoteEndpointProtocolVersion()).thenReturn(Commands.getCurrentProtocolVersion());
+ fixture.retries.get(3).run();
+ assertThat(fixture.requests).hasSize(1);
+ fixture.requests.get(0).complete(new byte[] {1});
+ assertThat(fixture.writes).hasValue(1);
+ }
+ }
+
+ @Test
+ public void testSchemaRetryRecoversAfterPendingMessageFails() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+ ProducerImpl<byte[]> producer = fixture.producer;
+ CompletableFuture<?> expired = producer.newMessage(Schema.STRING).value("expired").sendAsync();
+ fixture.requests.get(0).completeExceptionally(new PulsarClientException("registration failed"));
+ producer.failPendingMessages(null, new PulsarClientException.TimeoutException("send timed out"));
+ CompletableFuture<?> following = producer.newMessage().value(new byte[] {1}).sendAsync();
+ fixture.retries.get(0).run();
+ assertThat(expired).isCompletedExceptionally();
+ assertThat(following).isNotDone();
+ assertThat(producer.getState()).isEqualTo(HandlerState.State.Ready);
+ assertThat(fixture.writes).hasValue(1);
+ }
+ }
+
+ @DataProvider
+ public Object[][] schemaRetryConnectionChanges() {
+ return new Object[][] {{false}, {true}};
+ }
+
+ @Test(dataProvider = "schemaRetryConnectionChanges")
+ public void testSchemaRegistrationRetryIgnoresStaleConnection(boolean sameConnection) throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(true)) {
+ fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.rejectSchema();
+ assertThat(fixture.retries).hasSize(1);
+ // Re-registering on a shared TCP connection must invalidate the old epoch too.
+ fixture.producer.getConnectionHandler().switchClientCnx(sameConnection
+ ? fixture.cnx : mock(ClientCnx.class));
+ fixture.retries.get(0).run();
+ assertThat(fixture.requests).hasSize(1);
+ assertThat(fixture.retries).hasSize(1);
+ assertThat(fixture.producer.pendingMessages.size()).isEqualTo(1);
+ verify(fixture.cnx.channel(), never()).close();
+ }
+ }
+
+ @Test
+ public void testOrdinaryProducerStillFailsIncompatibleSchema() throws Exception {
+ try (SchemaRetryFixture fixture = new SchemaRetryFixture(false)) {
+ CompletableFuture<?> blocked = fixture.producer.newMessage(Schema.STRING).value("blocked").sendAsync();
+ fixture.rejectSchema();
+ assertThatThrownBy(blocked::join)
+ .hasCauseInstanceOf(PulsarClientException.IncompatibleSchemaException.class);
+ assertThat(fixture.retries).isEmpty();
+ assertThat(fixture.producer.pendingMessages.size()).isZero();
+ assertThat(fixture.producer.getState()).isEqualTo(HandlerState.State.Ready);
+ }
+ }
+
+ private static final class SchemaRetryFixture implements AutoCloseable {
+ private final ClientCnx cnx = mock(ClientCnx.class);
+ private final ProducerImpl<byte[]> producer;
+ private final List<CompletableFuture<byte[]>> requests = new ArrayList<>();
+ private final List<Runnable> retries = new ArrayList<>();
+ private final List<Long> retryDelays = new ArrayList<>();
+ private final List<ScheduledFuture<?>> retryTasks = new ArrayList<>();
+ private final AtomicInteger writes = new AtomicInteger();
+
+ private SchemaRetryFixture(boolean replicationProducer) {
+ PulsarClientImpl client = mockedPulsarClient();
+ when(client.getMemoryLimitController()).thenReturn(new MemoryLimitController(0));
+ when(client.getClientClock()).thenReturn(Clock.systemUTC());
+ client.getConfiguration().setInitialBackoffIntervalNanos(TimeUnit.MILLISECONDS.toNanos(100));
+ client.getConfiguration().setMaxBackoffIntervalNanos(TimeUnit.MILLISECONDS.toNanos(400));
+ ProducerConfigurationData conf = new ProducerConfigurationData();
+ conf.setBatchingEnabled(false);
+ conf.setSendTimeoutMs(0);
+ conf.setProducerName("schema-retry");
+ conf.setReplProducer(replicationProducer);
+ producer = constructProducer(client, conf);
+ EventLoop eventLoop = mock(EventLoop.class);
+ doAnswer(invocation -> {
+ invocation.<Runnable>getArgument(0).run();
+ return null;
+ }).when(eventLoop).execute(any(Runnable.class));
+ doAnswer(invocation -> {
+ retries.add(invocation.getArgument(0));
+ retryDelays.add(invocation.getArgument(1));
+ ScheduledFuture<?> task = mock(ScheduledFuture.class);
+ retryTasks.add(task);
+ return task;
+ }).when(eventLoop).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+ Channel channel = mock(Channel.class);
+ when(channel.eventLoop()).thenReturn(eventLoop);
+ ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
+ when(ctx.channel()).thenReturn(channel);
+ when(cnx.ctx()).thenReturn(ctx);
+ when(cnx.channel()).thenReturn(channel);
+ when(cnx.getRemoteEndpointProtocolVersion()).thenReturn(Commands.getCurrentProtocolVersion());
+ doAnswer(invocation -> {
+ ReferenceCountUtil.release(invocation.getArgument(0));
+ writes.incrementAndGet();
+ return null;
+ }).when(ctx).write(any(), any());
+ doAnswer(invocation -> {
+ ReferenceCountUtil.release(invocation.getArgument(0));
+ writes.incrementAndGet();
+ return null;
+ }).when(ctx).writeAndFlush(any(), any());
+ doAnswer(invocation -> {
+ ReferenceCountUtil.release(invocation.getArgument(0));
+ CompletableFuture<byte[]> response = new CompletableFuture<>();
+ requests.add(response);
+ return response;
+ }).when(cnx).sendGetOrCreateSchema(any(), anyLong());
+ doAnswer(invocation -> {
+ ReferenceCountUtil.release(invocation.getArgument(0));
+ return CompletableFuture.completedFuture(null);
+ }).when(cnx).sendRequestWithId(any(), anyLong());
+ producer.getConnectionHandler().switchClientCnx(cnx);
+ producer.setState(HandlerState.State.Ready);
+ }
+
+ private void rejectSchema() {
+ requests.get(requests.size() - 1).completeExceptionally(
+ new PulsarClientException.IncompatibleSchemaException("incompatible key schema"));
+ }
+
+ @Override
+ public void close() {
+ // Restore the mocked connection so close can clean up even after a stale-epoch test.
+ producer.getConnectionHandler().setClientCnx(cnx);
+ producer.closeAsync().join();
+ }
+ }
+
/**
* When encryption fails in any way, the partially built encrypted buffer must be released instead of
* leaking; the source payload stays with the caller (the batch container decides its fate).