You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/09/23 08:33:39 UTC

(pulsar) branch branch-4.2 updated (374569173a6a -> 8fd3aa10aae4)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git


    from 374569173a6a [fix][broker] Bound classic Key_Shared dispatcher replay queue look-ahead (#26677)
     new b8620060f9c2 [improve][broker] Disable subscription-thread dispatch by default (#26578)
     new 3335340e3466 [fix][client] Flush acknowledgment groups at the combined size limit (#26594)
     new c2bee15de40c [fix][client] Enable TableView compacted reads for short topic names (#26626)
     new b15171f82ded [fix][ml] Fix active cursor position tracking after untracking (#26629)
     new 5e87b6ae9414 [fix][build] Upgrade Conscrypt to 2.6.3 (#26660)
     new 8319acf0d4ed [fix][cli] Correct admin command documentation generation (#26649)
     new 8d785b7acbf7 [fix][ml] Fix backlog estimation during ledger rollover with pending writes (#26647)
     new 2a2c98ec379b [fix][meta] Track explicitly acquired underreplicated ledger locks (#26670)
     new eaaa8e842847 [fix] Fix ManagedLedgerImpl's pendingAddEntries leak. (#25240)
     new 9ec38639d2f5 [fix][ml] Fail in-flight adds when a managed ledger is terminated (#26678)
     new 5daa008bf63a [fix][ml] Fix batch ACK index loss when recovering cursor from MetadataStore (#26474)
     new 5d3cd8b5e7ec [fix][test] Join assignment tailer before asserting its last message ID (#26536)
     new 96e13f7b7658 [improve][broker] Conflate concurrent readMoreEntries calls (#26539)
     new 0d92b74c5e08 [fix][fn] Return descriptive errors for unknown JSON properties (#26441)
     new 371af5c03e8b [fix][client] Stop tracking ack timeouts on non-persistent topics (#26548)
     new 1815b88ebb5b [fix][broker] Avoid load shedding and metadata writes from a former leader (#26253)
     new fbad85453fed [fix][test] Avoid blocking common-pool workers in ledger shutdown test (#26560)
     new d0b8f53228b5 [fix][fn] Remove deprecated github.com/golang/protobuf dependency from pulsar-function-go (#26587)
     new 9e3cc4f1fa86 [fix][broker] Keep accepting Avro named type references written as objects (#26586)
     new d63678a4d9dd [fix][client] Drop a send receipt for a removed producer, not the connection (#26618)
     new 80df0506ebf3 [improve][broker] Reduce cache retention after reads and apply retention settings at startup (#26627)
     new 38384a36bc0f [fix][broker] Release the compaction buffers and permits on flush failures (#26576)
     new b8952393ff5c [fix][broker] Check topic permissions for partitions and subscriptions added to a transaction (#26644)
     new 71884df099c5 [fix][test] Isolate token CLI test output from background logging (#26657)
     new 304a8729494d [fix][broker] Apply the role logging anonymizer to topic authorization denial logs (#26642)
     new 320c5fcc4879 [fix][broker] Check topic permissions for binary GetSchema and GetOrCreateSchema (#26643)
     new 8fd3aa10aae4 [fix][broker] Preserve compaction state on ledger close failure (#26665)

The 27 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 conf/broker.conf                                   |   6 +-
 conf/standalone.conf                               |   4 +-
 distribution/server/src/assemble/LICENSE.bin.txt   |   2 +-
 distribution/shell/src/assemble/LICENSE.bin.txt    |   2 +-
 .../impl/ActiveManagedCursorContainerImpl.java     |  14 +-
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |  19 +-
 .../bookkeeper/mledger/impl/ManagedLedgerImpl.java |  72 +++++--
 ...ActiveManagedCursorContainerUntrackingTest.java | 163 ++++++++++++++++
 .../impl/ManagedCursorBatchAckRecoveryTest.java    | 203 ++++++++++++++++++++
 .../mledger/impl/ManagedLedgerBkTest.java          | 200 ++++++++++++++++++++
 .../impl/ManagedLedgerFactoryShutdownTest.java     |  21 +--
 .../bookkeeper/mledger/impl/ManagedLedgerTest.java |  88 +++++++++
 pom.xml                                            |   2 +-
 .../apache/pulsar/broker/ServiceConfiguration.java |   6 +-
 ...DefaultAuthenticationRoleLoggingAnonymizer.java |   3 +-
 ...ultAuthenticationRoleLoggingAnonymizerTest.java |  20 +-
 .../pulsar/broker/ManagedLedgerClientFactory.java  |   4 +
 .../loadbalance/impl/ModularLoadManagerImpl.java   | 165 ++++++++++++-----
 .../broker/service/BrokerServiceException.java     |   8 +
 .../apache/pulsar/broker/service/ServerCnx.java    | 125 ++++++++++---
 ...tractPersistentDispatcherMultipleConsumers.java |  57 ++++++
 .../PersistentDispatcherMultipleConsumers.java     |   5 +-
 ...rsistentDispatcherMultipleConsumersClassic.java |   3 +-
 .../broker/service/persistent/PersistentTopic.java |   6 +-
 .../schema/AvroSchemaBasedCompatibilityCheck.java  |  10 +-
 .../schema/JsonSchemaCompatibilityCheck.java       |   6 +-
 .../service/schema/SchemaRegistryServiceImpl.java  |   9 +-
 .../validator/StructSchemaDataValidator.java       |  19 +-
 .../client/impl/RawBatchMessageContainerImpl.java  |  12 +-
 .../compaction/StrategicTwoPhaseCompactor.java     |  29 ++-
 ...gedCursorBatchAckMetadataStoreRecoveryTest.java | 195 +++++++++++++++++++
 .../broker/ManagedLedgerClientFactoryTest.java     |  75 ++++++++
 .../impl/ModularLoadManagerImplTest.java           | 206 ++++++++++++++++++++-
 .../broker/service/BrokerBkEnsemblesTest.java      |  54 ++++++
 .../pulsar/broker/service/ServerCnxTest.java       | 175 +++++++++++++++++
 .../broker/service/SharedPulsarBaseTest.java       |   7 +
 .../persistent/CompactionConcurrencyTest.java      | 101 ++++++++++
 .../PersistentDispatcherMultipleConsumersTest.java | 162 ++++++++++++++++
 ...ntStickyKeyDispatcherMultipleConsumersTest.java |   4 +-
 .../schema/BaseAvroSchemaCompatibilityTest.java    |  20 ++
 .../schema/validator/SchemaDataValidatorTest.java  |  33 ++++
 .../broker/service/utils/ClientChannelHelper.java  |  12 ++
 .../api/ConsumerAcknowledgmentGroupSizeTest.java   | 126 +++++++++++++
 .../impl/RawBatchMessageContainerImplTest.java     |  37 ++++
 .../utils/auth/tokens/TokensCliUtilsTest.java      |  43 ++++-
 .../pulsar/admin/cli/CmdGenerateDocument.java      |  26 +--
 .../pulsar/admin/cli/TestCmdGenerateDocument.java  |  89 +++++++++
 .../org/apache/pulsar/client/impl/ClientCnx.java   |   2 +-
 .../apache/pulsar/client/impl/ConsumerBase.java    |  15 +-
 .../apache/pulsar/client/impl/ConsumerImpl.java    |  23 ++-
 .../PersistentAcknowledgmentsGroupingTracker.java  |  11 +-
 .../apache/pulsar/client/impl/TableViewImpl.java   |   3 +-
 .../pulsar/client/impl/ZeroQueueConsumerImpl.java  |   6 +-
 .../impl/schema/generic/GenericJsonRecord.java     |   3 +-
 .../pulsar/client/impl/schema/util/SchemaUtil.java |   3 +-
 .../apache/pulsar/client/impl/ClientCnxTest.java   |  27 +++
 .../pulsar/client/impl/ConsumerImplTest.java       |  72 ++++++-
 .../client/impl/TableViewBuilderImplTest.java      |   2 +
 .../pulsar/client/impl/TableViewImplTest.java      |  48 +++++
 .../client/impl/schema/util/SchemaUtilTest.java    |  19 ++
 .../pulsar/common/schema/AvroSchemaCompat.java     | 147 +++++++++++++++
 .../pulsar/common/schema/AvroSchemaCompatTest.java | 101 ++++++++++
 pulsar-function-go/go.mod                          |   1 -
 pulsar-function-go/pf/instance.go                  |   8 +-
 pulsar-function-go/pf/stats_test.go                |   6 +-
 .../pulsar/functions/worker/rest/Resources.java    |   3 +
 .../worker/FunctionAssignmentTailerTest.java       |  35 +---
 .../functions/worker/rest/ResourcesTest.java       |  17 +-
 .../PulsarLedgerUnderreplicationManager.java       |  15 +-
 .../LedgerUnderreplicationManagerTest.java         | 123 ++++++++++++
 70 files changed, 3099 insertions(+), 239 deletions(-)
 create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerUntrackingTest.java
 create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
 copy pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/AutoFailoverPolicyTypeTest.java => pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizerTest.java (51%)
 create mode 100644 pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckMetadataStoreRecoveryTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/ManagedLedgerClientFactoryTest.java
 create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAcknowledgmentGroupSizeTest.java
 create mode 100644 pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdGenerateDocument.java
 create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/schema/AvroSchemaCompat.java
 create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/schema/AvroSchemaCompatTest.java
 copy pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMetadataImplTest.java => pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/ResourcesTest.java (64%)


(pulsar) 10/27: [fix][ml] Fail in-flight adds when a managed ledger is terminated (#26678)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 9ec38639d2f5a701fab373637f3e5c9394c81aac
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Tue Sep 22 11:21:02 2026 -0700

    [fix][ml] Fail in-flight adds when a managed ledger is terminated (#26678)
    
    (cherry picked from commit a521e681f6610f8aaedad12d7d0e2ef24edc658e)
---
 .../bookkeeper/mledger/impl/ManagedLedgerImpl.java |  5 ++
 .../mledger/impl/ManagedLedgerBkTest.java          | 47 +++++++++++++++++++
 .../broker/service/BrokerBkEnsemblesTest.java      | 54 ++++++++++++++++++++++
 3 files changed, 106 insertions(+)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 4c52d640c547..25f23d82e9da 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -2000,6 +2000,11 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
         } else if (state.isFenced()) {
             clearPendingAddEntries(new ManagedLedgerFencedException("Managed ledger is fenced"));
             return;
+        } else if (state == State.Terminated) {
+            // The managed ledger was terminated during the write operation: the ledger got closed under the in-flight
+            // adds, and no new ledger will be created to retry them
+            clearPendingAddEntries(new ManagedLedgerTerminatedException("Managed ledger was terminated"));
+            return;
         } else {
             // In case we get multiple write errors for different outstanding write request, we should close the ledger
             // just once
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index 0d086b6be66c..5e19d48129c8 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -56,6 +56,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger;
 import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException;
+import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException;
 import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
 import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
 import org.apache.bookkeeper.mledger.Position;
@@ -63,6 +64,7 @@ import org.apache.bookkeeper.mledger.PositionFactory;
 import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager;
 import org.apache.bookkeeper.mledger.proto.PositionInfo;
 import org.apache.bookkeeper.mledger.util.ThrowableToStringUtil;
+import org.apache.bookkeeper.net.BookieId;
 import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
 import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
 import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
@@ -969,4 +971,49 @@ public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
         ledger1.close();
         factory.shutdown();
     }
+
+    /**
+     * Terminating a managed ledger closes the current BookKeeper ledger, and that close errors out the adds which are
+     * still in flight. These adds can never succeed, so they must be failed rather than left pending forever: a caller
+     * that tracks its in-flight writes (like the broker's PersistentTopic) would otherwise never see them complete.
+     */
+    @Test
+    public void testTerminateFailsInFlightAdds() throws Exception {
+        @Cleanup("shutdown")
+        ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+
+        ManagedLedgerConfig config = defaultConfig().setEnsembleSize(1).setWriteQuorumSize(1)
+                .setAckQuorumSize(1);
+        // A terminated ledger cannot be written to again: use a name that a retry of this test would not reuse
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config);
+        Position lastPersisted = ledger.addEntry("entry-0".getBytes());
+
+        // Suspend the bookie that stores the current ledger, so that the next add stays in flight
+        BookieId bookie = ledger.currentLedger.getLedgerMetadata().getAllEnsembles().firstEntry().getValue().get(0);
+        CountDownLatch resumeBookie = new CountDownLatch(1);
+        sleepBookie(bookie, resumeBookie);
+        try {
+            CompletableFuture<ManagedLedgerException> inFlightAddFailure = new CompletableFuture<>();
+            ledger.asyncAddEntry("entry-1".getBytes(), new AddEntryCallback() {
+                @Override
+                public void addComplete(Position position, ByteBuf entryData, Object ctx) {
+                    inFlightAddFailure.completeExceptionally(
+                            new AssertionError("The in-flight add should not succeed: " + position));
+                }
+
+                @Override
+                public void addFailed(ManagedLedgerException exception, Object ctx) {
+                    inFlightAddFailure.complete(exception);
+                }
+            }, null);
+            Awaitility.await().until(() -> !ledger.pendingAddEntries.isEmpty());
+
+            assertEquals(ledger.terminate(), lastPersisted);
+
+            assertTrue(inFlightAddFailure.get(10, TimeUnit.SECONDS) instanceof ManagedLedgerTerminatedException);
+            assertTrue(ledger.pendingAddEntries.isEmpty());
+        } finally {
+            resumeBookie.countDown();
+        }
+    }
 }
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTest.java
index 646da7d1fb43..64bcf4d1e0e8 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTest.java
@@ -26,6 +26,7 @@ import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
 import static org.testng.Assert.fail;
 import com.google.common.collect.Sets;
+import io.netty.buffer.Unpooled;
 import io.netty.channel.EventLoopGroup;
 import io.netty.util.concurrent.DefaultThreadFactory;
 import java.io.Closeable;
@@ -52,6 +53,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
 import org.apache.bookkeeper.mledger.proto.ManagedLedgerInfo.LedgerInfo;
 import org.apache.bookkeeper.util.StringUtils;
 import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.client.api.Consumer;
@@ -61,6 +63,7 @@ import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionInitialPosition;
 import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
 import org.apache.pulsar.common.util.netty.EventLoopUtil;
@@ -647,4 +650,55 @@ public class BrokerBkEnsemblesTest extends BkEnsemblesTestBase {
         bkClient2.close();
     }
 
+    /**
+     * Terminating a topic closes the current ledger under the publishes that are still in flight. The topic must not
+     * stay fenced afterwards: a terminated topic still has to be drained, so consumers must be able to subscribe to it.
+     */
+    @Test
+    public void testConsumerCanSubscribeAfterTerminateWithInFlightPublish() throws Exception {
+        final String namespace = BrokerTestUtil.newUniqueName("prop/ns");
+        final String topic = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp");
+        admin.namespaces().createNamespace(namespace);
+        admin.topics().createNonPartitionedTopic(topic);
+
+        @Cleanup
+        PulsarClient client = PulsarClient.builder()
+                .serviceUrl(pulsar.getBrokerServiceUrl())
+                .statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        Producer<String> producer = client.newProducer(Schema.STRING)
+                .enableBatching(false)
+                .topic(topic)
+                .create();
+        producer.send("1");
+        PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService()
+                .getTopic(topic, false).join().get();
+
+        // Delay the bookie responses, so that the next publish is still in flight when the topic gets terminated.
+        Closeable cancellation = injectBKServerDelayForCurrentLedger(topic, 10, TimeUnit.SECONDS, 0);
+        producer.sendAsync("2");
+        Awaitility.await().untilAsserted(() -> assertEquals(persistentTopic.getPendingWriteOps().get(), 1));
+
+        CompletableFuture<MessageId> terminated = persistentTopic.terminate();
+        // Under load, more publishes reach the terminated topic before its producers get disconnected.
+        CompletableFuture<Exception> latePublish = new CompletableFuture<>();
+        persistentTopic.publishMessage(Unpooled.wrappedBuffer("3".getBytes()),
+                (e, ledgerId, entryId) -> latePublish.complete(e));
+        terminated.get(10, TimeUnit.SECONDS);
+        assertTrue(latePublish.get(10, TimeUnit.SECONDS) instanceof TopicTerminatedException);
+        cancellation.close();
+
+        // The publish that was in flight is failed, rather than being left pending forever.
+        Awaitility.await().untilAsserted(() -> assertEquals(persistentTopic.getPendingWriteOps().get(), 0));
+
+        Consumer<String> consumer = client.newConsumer(Schema.STRING)
+                .topic(topic)
+                .subscriptionName("s1")
+                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+                .subscribe();
+        Message<String> msg = consumer.receive(10, TimeUnit.SECONDS);
+        assertNotNull(msg);
+        assertEquals(msg.getValue(), "1");
+    }
+
 }


(pulsar) 23/27: [fix][broker] Check topic permissions for partitions and subscriptions added to a transaction (#26644)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit b8952393ff5cf0cbd3437d38de9b80992f6f20e2
Author: kannar <ka...@gmail.com>
AuthorDate: Sat Sep 19 16:07:27 2026 +0200

    [fix][broker] Check topic permissions for partitions and subscriptions added to a transaction (#26644)
    
    (cherry picked from commit 28eea815e5c223491537f69dbf2b52a9b5563075)
---
 .../broker/service/BrokerServiceException.java     |  8 +++
 .../apache/pulsar/broker/service/ServerCnx.java    | 59 ++++++++++++++--
 .../pulsar/broker/service/ServerCnxTest.java       | 80 ++++++++++++++++++++++
 3 files changed, 142 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
index 43a95cc86016..95c5550d3b21 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java
@@ -188,6 +188,12 @@ public class BrokerServiceException extends Exception {
         }
     }
 
+    public static class NotAuthorizedException extends BrokerServiceException {
+        public NotAuthorizedException(String msg) {
+            super(msg);
+        }
+    }
+
     public static class SubscriptionInvalidCursorPosition extends BrokerServiceException {
         public SubscriptionInvalidCursorPosition(String msg) {
             super(msg);
@@ -277,6 +283,8 @@ public class BrokerServiceException extends Exception {
             return ServerError.InvalidTxnStatus;
         } else if (t instanceof NotAllowedException) {
             return ServerError.NotAllowedError;
+        } else if (t instanceof NotAuthorizedException) {
+            return ServerError.AuthorizationError;
         } else if (t instanceof ProducerFencedException) {
             return ServerError.ProducerFenced;
         } else if (t instanceof TransactionConflictException) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 8c214deff45f..852494e58e0b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -2965,9 +2965,10 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                     if (!isOwner) {
                         return failedFutureTxnNotOwned(txnID);
                     }
-                    return transactionMetadataStoreService
-                            .addProducedPartitionToTxn(txnID, partitionsList);
+                    return checkTxnPartitionsAuthorized(partitionsList);
                 })
+                .thenCompose(__ -> transactionMetadataStoreService
+                        .addProducedPartitionToTxn(txnID, partitionsList))
                 .whenComplete((v, ex) -> {
                     if (ex == null) {
                         if (log.isDebugEnabled()) {
@@ -2988,10 +2989,57 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                 });
     }
 
+    /**
+     * The transaction coordinator ends a transaction on its registered partitions with its own identity, so a
+     * client may only register partitions it could produce to.
+     */
+    private CompletableFuture<Void> checkTxnPartitionsAuthorized(List<String> partitions) {
+        List<CompletableFuture<Void>> checks = new ArrayList<>(partitions.size());
+        for (String partition : partitions) {
+            checks.add(checkTxnParticipantAuthorized(partition, null, TopicOperation.PRODUCE));
+        }
+        return FutureUtil.waitForAll(checks);
+    }
+
+    /**
+     * The transaction coordinator ends a transaction on its registered subscriptions with its own identity, so a
+     * client may only register subscriptions it could consume from.
+     */
+    private CompletableFuture<Void> checkTxnSubscriptionsAuthorized(
+            List<org.apache.pulsar.common.api.proto.Subscription> subscriptions) {
+        List<CompletableFuture<Void>> checks = new ArrayList<>(subscriptions.size());
+        for (org.apache.pulsar.common.api.proto.Subscription subscription : subscriptions) {
+            checks.add(checkTxnParticipantAuthorized(subscription.getTopic(), subscription.getSubscription(),
+                    TopicOperation.CONSUME));
+        }
+        return FutureUtil.waitForAll(checks);
+    }
+
+    private CompletableFuture<Void> checkTxnParticipantAuthorized(String topic, String subscription,
+                                                                  TopicOperation operation) {
+        if (!service.isAuthorizationEnabled()) {
+            return CompletableFuture.completedFuture(null);
+        }
+        final TopicName topicName;
+        try {
+            topicName = TopicName.get(topic);
+        } catch (IllegalArgumentException e) {
+            return CompletableFuture.failedFuture(new BrokerServiceException.NotAllowedException(
+                    "Invalid topic name " + topic + ": " + e.getMessage()));
+        }
+        CompletableFuture<Boolean> isAuthorized = subscription == null
+                ? isTopicOperationAllowed(topicName, operation, authenticationData, originalAuthData)
+                : isTopicOperationAllowed(topicName, subscription, operation);
+        return isAuthorized.thenCompose(authorized -> authorized
+                ? CompletableFuture.<Void>completedFuture(null)
+                : CompletableFuture.failedFuture(new BrokerServiceException.NotAuthorizedException(
+                        "Client is not authorized to " + operation + " on " + topic)));
+    }
+
     private CompletableFuture<Void> failedFutureTxnNotOwned(TxnID txnID) {
         String msg = String.format(
                 "Client (%s) is neither the owner of the transaction %s nor a super user",
-                getPrincipal(), txnID
+                authenticationRoleLoggingAnonymizer.anonymize(getPrincipal()), txnID
         );
         log.warn("[{}] {}", remoteAddress, msg);
         return CompletableFuture.failedFuture(new CoordinatorException.TransactionNotFoundException(msg));
@@ -3310,9 +3358,10 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
                     if (!isOwner) {
                         return failedFutureTxnNotOwned(txnID);
                     }
-                    return transactionMetadataStoreService.addAckedPartitionToTxn(txnID,
-                            MLTransactionMetadataStore.subscriptionToTxnSubscription(subscriptionsList));
+                    return checkTxnSubscriptionsAuthorized(subscriptionsList);
                 })
+                .thenCompose(__ -> transactionMetadataStoreService.addAckedPartitionToTxn(txnID,
+                        MLTransactionMetadataStore.subscriptionToTxnSubscription(subscriptionsList)))
                 .whenComplete((v, ex) -> {
                     if (ex == null) {
                         if (log.isDebugEnabled()) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 24d389bd93c6..968fd442005e 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -3814,6 +3814,86 @@ public class ServerCnxTest {
         channel.finish();
     }
 
+    private AuthorizationService mockTopicAuthorization(boolean allowed) {
+        AuthorizationService authorizationService = mock(AuthorizationService.class);
+        doReturn(CompletableFuture.completedFuture(allowed)).when(authorizationService)
+                .allowTopicOperationAsync(Mockito.any(),
+                        Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
+        doReturn(authorizationService).when(brokerService).getAuthorizationService();
+        svcConfig.setAuthenticationEnabled(true);
+        svcConfig.setAuthorizationEnabled(true);
+        return authorizationService;
+    }
+
+    @Test(timeOut = 30000)
+    public void testAddPartitionToTxnRefusedWhenOnePartitionIsDenied() throws Exception {
+        AuthorizationService authorizationService = mockTopicAuthorization(true);
+        doReturn(CompletableFuture.completedFuture(false)).when(authorizationService)
+                .allowTopicOperationAsync(eq(TopicName.get("tenant/ns/denied")),
+                        Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
+        final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class);
+        when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true));
+        when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore);
+        svcConfig.setTransactionCoordinatorEnabled(true);
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newAddPartitionToTxn(89L, 1L, 12L,
+                List.of("tenant/ns/allowed", "tenant/ns/denied")));
+        CommandAddPartitionToTxnResponse response = (CommandAddPartitionToTxnResponse) getResponse();
+
+        assertEquals(response.getError(), ServerError.AuthorizationError);
+        verify(txnStore, never()).addProducedPartitionToTxn(any(TxnID.class), any());
+
+        channel.finish();
+    }
+
+    @Test(timeOut = 30000)
+    public void testAddPartitionToTxnRequiresProducePermission() throws Exception {
+        AuthorizationService authorizationService = mockTopicAuthorization(false);
+        final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class);
+        when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true));
+        when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore);
+        svcConfig.setTransactionCoordinatorEnabled(true);
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newAddPartitionToTxn(89L, 1L, 12L, List.of("tenant/ns/topic1")));
+        CommandAddPartitionToTxnResponse response = (CommandAddPartitionToTxnResponse) getResponse();
+
+        assertEquals(response.getError(), ServerError.AuthorizationError);
+        verify(authorizationService, times(1)).allowTopicOperationAsync(eq(TopicName.get("tenant/ns/topic1")),
+                eq(TopicOperation.PRODUCE), any(), any(), any(), any());
+        verify(txnStore, never()).addProducedPartitionToTxn(any(TxnID.class), any());
+
+        channel.finish();
+    }
+
+    @Test(timeOut = 30000)
+    public void testAddSubscriptionToTxnRequiresConsumePermission() throws Exception {
+        AuthorizationService authorizationService = mockTopicAuthorization(false);
+        final TransactionMetadataStoreService txnStore = mock(TransactionMetadataStoreService.class);
+        when(txnStore.verifyTxnOwnership(any(), any())).thenReturn(CompletableFuture.completedFuture(true));
+        when(pulsar.getTransactionMetadataStoreService()).thenReturn(txnStore);
+        svcConfig.setTransactionCoordinatorEnabled(true);
+        resetChannel();
+        setChannelConnected();
+        final Subscription sub = new Subscription();
+        sub.setTopic("topic1");
+        sub.setSubscription("sub1");
+
+        channel.writeInbound(Commands.newAddSubscriptionToTxn(89L, 1L, 12L, List.of(sub)));
+        CommandAddSubscriptionToTxnResponse response = (CommandAddSubscriptionToTxnResponse) getResponse();
+
+        assertEquals(response.getError(), ServerError.AuthorizationError);
+        verify(authorizationService, times(1)).allowTopicOperationAsync(eq(TopicName.get("topic1")),
+                eq(TopicOperation.CONSUME), any(), any(), any(), argThat(arg ->
+                        arg instanceof AuthenticationDataSubscription
+                                && "sub1".equals(((AuthenticationDataSubscription) arg).getSubscription())));
+        verify(txnStore, never()).addAckedPartitionToTxn(any(TxnID.class), any());
+
+        channel.finish();
+    }
 
     @Test(timeOut = 30000)
     public void sendEndTxnResponse() throws Exception {


(pulsar) 08/27: [fix][meta] Track explicitly acquired underreplicated ledger locks (#26670)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 2a2c98ec379b6b9694e8630de75810ff273af8a2
Author: void-ptr974 <vo...@protonmail.com>
AuthorDate: Wed Sep 23 02:19:44 2026 +0800

    [fix][meta] Track explicitly acquired underreplicated ledger locks (#26670)
    
    (cherry picked from commit 2857cc37e9bece2ecea532d44c4d8710ff0c070e)
---
 .../PulsarLedgerUnderreplicationManager.java       |  15 ++-
 .../LedgerUnderreplicationManagerTest.java         | 123 +++++++++++++++++++++
 2 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
index f56f6e10f829..8862eece7452 100644
--- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
+++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java
@@ -84,9 +84,9 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
 
     private static class Lock {
         private final String lockPath;
-        private final long ledgerNodeVersion;
+        private final Optional<Long> ledgerNodeVersion;
 
-        Lock(String lockPath, long ledgerNodeVersion) {
+        Lock(String lockPath, Optional<Long> ledgerNodeVersion) {
             this.lockPath = lockPath;
             this.ledgerNodeVersion = ledgerNodeVersion;
         }
@@ -95,7 +95,7 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
             return lockPath;
         }
 
-        long getLedgerNodeVersion() {
+        Optional<Long> getLedgerNodeVersion() {
             return ledgerNodeVersion;
         }
     }
@@ -422,6 +422,9 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
     public void acquireUnderreplicatedLedger(long ledgerId) throws ReplicationException {
         try {
             internalAcquireUnderreplicatedLedger(ledgerId);
+            String lockPath = getUrLedgerLockPath(urLockPath, ledgerId);
+            // Explicit acquisition holds only the lock, without claiming an underreplication record version.
+            heldLocks.put(ledgerId, new Lock(lockPath, Optional.empty()));
         } catch (ExecutionException | TimeoutException | InterruptedException e) {
             throw new ReplicationException.UnavailableException("Failed to acuire under-replicated ledger", e);
         }
@@ -441,8 +444,8 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
         }
         try {
             Lock l = heldLocks.get(ledgerId);
-            if (l != null) {
-                store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion()))
+            if (l != null && l.getLedgerNodeVersion().isPresent()) {
+                store.delete(getUrLedgerPath(ledgerId), l.getLedgerNodeVersion())
                         .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS);
                 if (store instanceof ZKMetadataStore
                         || store instanceof DualMetadataStore
@@ -592,7 +595,7 @@ public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicati
                     long ledgerId = getLedgerId(tryChild);
                     internalAcquireUnderreplicatedLedger(ledgerId);
                     String lockPath = getUrLedgerLockPath(urLockPath, ledgerId);
-                    heldLocks.put(ledgerId, new Lock(lockPath, optRes.get().getStat().getVersion()));
+                    heldLocks.put(ledgerId, new Lock(lockPath, Optional.of(optRes.get().getStat().getVersion())));
                     return ledgerId;
                 } catch (ExecutionException ee) {
                     if (ee.getCause() instanceof MetadataStoreException.BadVersionException) {
diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java
index 1c9231c3084f..37b034fd95fd 100644
--- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java
+++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java
@@ -18,6 +18,8 @@
  */
 package org.apache.pulsar.metadata.bookkeeper;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
@@ -136,6 +138,127 @@ public class LedgerUnderreplicationManagerTest extends BaseMetadataStoreTest {
         }
     }
 
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testExplicitLockRelease(String provider, Supplier<String> urlSupplier) throws Exception {
+        methodSetup(urlSupplier);
+        long ledgerId = 123L;
+        try (var other = lmf.newLedgerUnderreplicationManager()) {
+            assertThat(lum.getLedgerUnreplicationInfo(ledgerId)).isNull();
+            lum.acquireUnderreplicatedLedger(ledgerId);
+            assertThat(other.isLedgerBeingReplicated(ledgerId)).isTrue();
+
+            lum.releaseUnderreplicatedLedger(ledgerId);
+            assertThat(other.isLedgerBeingReplicated(ledgerId)).isFalse();
+            other.acquireUnderreplicatedLedger(ledgerId);
+
+            // Releasing again must not remove the new owner's lock.
+            lum.releaseUnderreplicatedLedger(ledgerId);
+            assertThat(other.isLedgerBeingReplicated(ledgerId)).isTrue();
+            other.releaseUnderreplicatedLedger(ledgerId);
+            assertThat(lum.isLedgerBeingReplicated(ledgerId)).isFalse();
+        }
+    }
+
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testExplicitLockCloseWithSharedStore(String provider, Supplier<String> urlSupplier) throws Exception {
+        methodSetup(urlSupplier);
+        long polledLedgerId = 123L;
+        long explicitLedgerId = 456L;
+        try (var other = lmf.newLedgerUnderreplicationManager()) {
+            lum.markLedgerUnderreplicated(polledLedgerId, "bookie:3181");
+            assertThat(lum.pollLedgerToRereplicate()).isEqualTo(polledLedgerId);
+            lum.acquireUnderreplicatedLedger(explicitLedgerId);
+
+            lum.close();
+
+            // Both managers share the same store, which remains open after lum.close().
+            assertThat(other.isLedgerBeingReplicated(polledLedgerId)).isFalse();
+            assertThat(other.isLedgerBeingReplicated(explicitLedgerId)).isFalse();
+            other.acquireUnderreplicatedLedger(explicitLedgerId);
+            assertThat(other.pollLedgerToRereplicate()).isEqualTo(polledLedgerId);
+            other.releaseUnderreplicatedLedger(explicitLedgerId);
+            other.markLedgerReplicated(polledLedgerId);
+        }
+    }
+
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testExplicitLockMarkPreservesUnderreplicatedRecord(String provider, Supplier<String> urlSupplier)
+            throws Exception {
+        methodSetup(urlSupplier);
+        long ledgerId = 123L;
+        lum.markLedgerUnderreplicated(ledgerId, "bookie:3181");
+        String ledgerPath = PulsarLedgerUnderreplicationManager.getUrLedgerPath(urLedgerPath, ledgerId);
+        GetResult original = store.get(ledgerPath).get(10, TimeUnit.SECONDS).orElseThrow();
+
+        lum.acquireUnderreplicatedLedger(ledgerId);
+        lum.markLedgerReplicated(ledgerId);
+
+        GetResult retained = store.get(ledgerPath).get(10, TimeUnit.SECONDS).orElseThrow();
+        assertThat(retained.getValue()).isEqualTo(original.getValue());
+        assertThat(retained.getStat().getVersion()).isEqualTo(original.getStat().getVersion());
+        assertThat(lum.isLedgerBeingReplicated(ledgerId)).isFalse();
+        try (var other = lmf.newLedgerUnderreplicationManager()) {
+            assertThat(other.pollLedgerToRereplicate()).isEqualTo(ledgerId);
+            other.markLedgerReplicated(ledgerId);
+            assertThat(other.getLedgerUnreplicationInfo(ledgerId)).isNull();
+            assertThat(other.isLedgerBeingReplicated(ledgerId)).isFalse();
+        }
+    }
+
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testExplicitLockMarkWithoutUnderreplicatedRecord(String provider, Supplier<String> urlSupplier)
+            throws Exception {
+        methodSetup(urlSupplier);
+        long ledgerId = 123L;
+        lum.acquireUnderreplicatedLedger(ledgerId);
+        lum.markLedgerReplicated(ledgerId);
+
+        assertThat(lum.getLedgerUnreplicationInfo(ledgerId)).isNull();
+        assertThat(lum.isLedgerBeingReplicated(ledgerId)).isFalse();
+        try (var other = lmf.newLedgerUnderreplicationManager()) {
+            other.acquireUnderreplicatedLedger(ledgerId);
+            other.releaseUnderreplicatedLedger(ledgerId);
+        }
+    }
+
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testFailedExplicitLockAcquireCannotReleaseOwner(String provider, Supplier<String> urlSupplier)
+            throws Exception {
+        methodSetup(urlSupplier);
+        long ledgerId = 123L;
+        lum.markLedgerUnderreplicated(ledgerId, "bookie:3181");
+        assertThat(lum.pollLedgerToRereplicate()).isEqualTo(ledgerId);
+        try (var other = lmf.newLedgerUnderreplicationManager()) {
+            assertThatThrownBy(() -> other.acquireUnderreplicatedLedger(ledgerId))
+                    .isInstanceOf(UnavailableException.class);
+            other.releaseUnderreplicatedLedger(ledgerId);
+            assertThat(lum.isLedgerBeingReplicated(ledgerId)).isTrue();
+            other.markLedgerReplicated(ledgerId);
+            assertThat(lum.getLedgerUnreplicationInfo(ledgerId)).isNotNull();
+            other.close();
+            assertThat(lum.isLedgerBeingReplicated(ledgerId)).isTrue();
+        }
+        lum.markLedgerReplicated(ledgerId);
+        assertThat(lum.getLedgerUnreplicationInfo(ledgerId)).isNull();
+        assertThat(lum.isLedgerBeingReplicated(ledgerId)).isFalse();
+    }
+
+    @Test(timeOut = 30000, dataProvider = "impl")
+    public void testFailedExplicitLockAcquirePreservesPolledVersion(String provider, Supplier<String> urlSupplier)
+            throws Exception {
+        methodSetup(urlSupplier);
+        long ledgerId = 123L;
+        lum.markLedgerUnderreplicated(ledgerId, "bookie:3181");
+        assertThat(lum.pollLedgerToRereplicate()).isEqualTo(ledgerId);
+        assertThatThrownBy(() -> lum.acquireUnderreplicatedLedger(ledgerId))
+                .isInstanceOf(UnavailableException.class);
+
+        lum.markLedgerReplicated(ledgerId);
+
+        assertThat(lum.getLedgerUnreplicationInfo(ledgerId)).isNull();
+        assertThat(lum.isLedgerBeingReplicated(ledgerId)).isFalse();
+    }
+
     /**
      * Test basic interactions with the ledger underreplication
      * manager.


(pulsar) 13/27: [improve][broker] Conflate concurrent readMoreEntries calls (#26539)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 96e13f7b76585ced64f9530164d5f344309a0d7b
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Sep 11 19:18:35 2026 +0300

    [improve][broker] Conflate concurrent readMoreEntries calls (#26539)
    
    (cherry picked from commit d5dec18cba9908318f9dfb0d097fef6e1832087b)
---
 ...tractPersistentDispatcherMultipleConsumers.java |  57 ++++++++
 .../PersistentDispatcherMultipleConsumers.java     |   5 +-
 ...rsistentDispatcherMultipleConsumersClassic.java |   3 +-
 .../PersistentDispatcherMultipleConsumersTest.java | 162 +++++++++++++++++++++
 ...ntStickyKeyDispatcherMultipleConsumersTest.java |   4 +-
 5 files changed, 226 insertions(+), 5 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
index 3a63dc659470..a0c0c0358646 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service.persistent;
 
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
@@ -32,6 +33,11 @@ import org.apache.pulsar.common.policies.data.stats.TopicMetricBean;
 
 public abstract class AbstractPersistentDispatcherMultipleConsumers extends AbstractDispatcherMultipleConsumers
         implements Dispatcher, AsyncCallbacks.ReadEntriesCallback {
+    private static final int IDLE = 0;
+    private static final int RUNNING = 1;
+    private static final int RUNNING_REQUESTED = 2;
+    private final AtomicInteger readMoreEntriesState = new AtomicInteger(IDLE);
+
     public AbstractPersistentDispatcherMultipleConsumers(Subscription subscription,
                                                          ServiceConfiguration serviceConfig) {
         super(subscription, serviceConfig);
@@ -41,6 +47,57 @@ public abstract class AbstractPersistentDispatcherMultipleConsumers extends Abst
 
     public abstract void readMoreEntriesAsync();
 
+    /**
+     * Conflate concurrent and reentrant requests into follow-up passes driven by the owning caller.
+     * If a pass fails, drain registered follow-ups before propagating the first failure to the owner.
+     */
+    public final void readMoreEntries() {
+        for (;;) {
+            int state = readMoreEntriesState.get();
+            if (state == IDLE) {
+                if (readMoreEntriesState.compareAndSet(IDLE, RUNNING)) {
+                    break;
+                }
+            } else if (state == RUNNING) {
+                if (readMoreEntriesState.compareAndSet(RUNNING, RUNNING_REQUESTED)) {
+                    return;
+                }
+            } else {
+                // A follow-up pass is already requested.
+                return;
+            }
+        }
+        Throwable failure = null;
+        for (;;) {
+            // Requests received before this pass are covered by this pass.
+            readMoreEntriesState.set(RUNNING);
+            try {
+                internalReadMoreEntries();
+            } catch (RuntimeException | Error t) {
+                // Keep ownership so a failure cannot discard an already-registered follow-up.
+                if (failure == null) {
+                    failure = t;
+                } else if (failure != t) {
+                    failure.addSuppressed(t);
+                }
+            }
+            if (readMoreEntriesState.compareAndSet(RUNNING, IDLE)) {
+                break;
+            }
+            // A request arrived during the pass. Run another pass without recursion, even after a failure.
+        }
+        if (failure instanceof RuntimeException runtimeException) {
+            throw runtimeException;
+        } else if (failure instanceof Error error) {
+            throw error;
+        }
+    }
+
+    /**
+     * Execute one read pass. Implementations must synchronize on this dispatcher to protect read state.
+     */
+    protected abstract void internalReadMoreEntries();
+
     public abstract String getName();
 
     public abstract boolean isBlockedDispatcherOnUnackedMsgs();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 938f58b9440a..009af8299543 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -345,7 +345,8 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
         }
     }
 
-    public synchronized void readMoreEntries() {
+    @Override
+    protected synchronized void internalReadMoreEntries() {
         if (cursor.isClosed()) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
@@ -373,7 +374,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis
             return;
         }
 
-        // increment the counter for readMoreEntries calls, to track the number of times readMoreEntries is called
+        // Count executed passes, not conflated requests, for pending-read rescheduling.
         readMoreEntriesCallCount++;
 
         // totalAvailablePermits may be updated by other threads
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
index dcb2209f4b1d..a0bb11bdbb08 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java
@@ -315,7 +315,8 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis
         topic.getBrokerService().executor().execute(this::readMoreEntries);
     }
 
-    public synchronized void readMoreEntries() {
+    @Override
+    protected synchronized void internalReadMoreEntries() {
         if (cursor.isClosed()) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Cursor is already closed, skipping read more entries.", cursor.getName());
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
index 79fd3b03d1fc..5843a77546bc 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersTest.java
@@ -18,11 +18,14 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import com.carrotsearch.hppc.ObjectSet;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -43,12 +46,171 @@ import org.apache.pulsar.common.api.proto.MessageMetadata;
 import org.awaitility.reflect.WhiteboxImpl;
 import org.mockito.Mockito;
 import org.testng.Assert;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 @Slf4j
 @Test(groups = "broker-api")
 public class PersistentDispatcherMultipleConsumersTest extends SharedPulsarBaseTest {
 
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesConflatesConcurrentRequests(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        CountDownLatch[] passStarted = {new CountDownLatch(1), new CountDownLatch(1)};
+        CountDownLatch[] releasePass = {new CountDownLatch(1), new CountDownLatch(1)};
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            assertThat(Thread.holdsLock(dispatcher)).isTrue();
+            int pass = passes.getAndIncrement();
+            if (pass < passStarted.length) {
+                passStarted[pass].countDown();
+                assertThat(releasePass[pass].await(10, TimeUnit.SECONDS)).isTrue();
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        try {
+            Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+            for (int pass = 0; pass < passStarted.length; pass++) {
+                assertThat(passStarted[pass].await(10, TimeUnit.SECONDS)).isTrue();
+                // All callers must return while the owner still holds the dispatcher monitor.
+                executor.submit(() -> {
+                    for (int request = 0; request < 100; request++) {
+                        dispatcher.readMoreEntries();
+                    }
+                }).get(10, TimeUnit.SECONDS);
+                assertThat(passes.get()).isEqualTo(pass + 1);
+                releasePass[pass].countDown();
+            }
+            owner.get(10, TimeUnit.SECONDS);
+            assertThat(passes.get()).isEqualTo(3);
+            dispatcher.readMoreEntries();
+            assertThat(passes.get()).isEqualTo(4);
+        } finally {
+            for (CountDownLatch latch : releasePass) {
+                latch.countDown();
+            }
+            executor.shutdownNow();
+            assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+        }
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesConflatesReentrantRequests(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        AtomicInteger passes = new AtomicInteger();
+        AtomicInteger depth = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            assertThat(depth.incrementAndGet()).isEqualTo(1);
+            try {
+                if (passes.incrementAndGet() < 1000) {
+                    dispatcher.readMoreEntries();
+                    dispatcher.readMoreEntries();
+                }
+                return true;
+            } finally {
+                depth.decrementAndGet();
+            }
+        }).when(cursor).isClosed();
+
+        dispatcher.readMoreEntries();
+        assertThat(passes.get()).isEqualTo(1000);
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRecoversAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        Mockito.doThrow(failure).when(cursor).isClosed();
+        assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+
+        Mockito.doReturn(true).when(cursor).isClosed();
+        Mockito.clearInvocations(cursor);
+        dispatcher.readMoreEntries();
+        Mockito.verify(cursor).isClosed();
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRunsReentrantRequestsAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        IllegalStateException followUpFailure = new IllegalStateException("follow-up read failed");
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            int pass = passes.incrementAndGet();
+            if (pass <= 3) {
+                dispatcher.readMoreEntries();
+                // Reusing the first exception must not cause self-suppression.
+                throw pass == 2 ? followUpFailure : failure;
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        assertThatThrownBy(dispatcher::readMoreEntries).isSameAs(failure);
+        assertThat(passes.get()).as("Registered follow-ups run before propagating the failure").isEqualTo(4);
+        assertThat(failure.getSuppressed()).containsExactly(followUpFailure);
+        dispatcher.readMoreEntries();
+        assertThat(passes.get()).isEqualTo(5);
+    }
+
+    @Test(timeOut = 30_000, dataProvider = "readConflationDispatcherTypes")
+    public void testReadMoreEntriesRunsConcurrentRequestAfterFailure(boolean classic) throws Exception {
+        ManagedCursor cursor = Mockito.mock(ManagedCursor.class);
+        AbstractPersistentDispatcherMultipleConsumers dispatcher = createReadConflationDispatcher(cursor, classic);
+        IllegalStateException failure = new IllegalStateException("read failed");
+        CountDownLatch passStarted = new CountDownLatch(1);
+        CountDownLatch releasePass = new CountDownLatch(1);
+        AtomicInteger passes = new AtomicInteger();
+        Mockito.doAnswer(inv -> {
+            if (passes.incrementAndGet() == 1) {
+                passStarted.countDown();
+                assertThat(releasePass.await(10, TimeUnit.SECONDS)).isTrue();
+                throw failure;
+            }
+            return true;
+        }).when(cursor).isClosed();
+
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        try {
+            Future<?> owner = executor.submit(dispatcher::readMoreEntries);
+            assertThat(passStarted.await(10, TimeUnit.SECONDS)).isTrue();
+            executor.submit(dispatcher::readMoreEntries).get(10, TimeUnit.SECONDS);
+            releasePass.countDown();
+            assertThatThrownBy(() -> owner.get(10, TimeUnit.SECONDS)).hasCause(failure);
+            assertThat(passes.get()).as("The registered follow-up runs without a new caller").isEqualTo(2);
+            dispatcher.readMoreEntries();
+            assertThat(passes.get()).isEqualTo(3);
+        } finally {
+            releasePass.countDown();
+            executor.shutdownNow();
+            assertThat(executor.awaitTermination(10, TimeUnit.SECONDS)).isTrue();
+        }
+    }
+
+    @DataProvider
+    public Object[][] readConflationDispatcherTypes() {
+        return new Object[][] {{false}, {true}};
+    }
+
+    private AbstractPersistentDispatcherMultipleConsumers createReadConflationDispatcher(ManagedCursor cursor,
+                                                                                       boolean classic)
+            throws Exception {
+        String topicName = newTopicName();
+        admin.topics().createNonPartitionedTopic(topicName);
+        admin.topics().createSubscription(topicName, "s1", MessageId.earliest);
+        PersistentTopic topic = (PersistentTopic) getTopic(topicName, false).join().orElseThrow();
+        Mockito.doReturn("s1").when(cursor).getName();
+        Subscription subscription = Mockito.mock(PersistentSubscription.class);
+        Mockito.doReturn(topic).when(subscription).getTopic();
+        return classic ? new PersistentDispatcherMultipleConsumersClassic(topic, cursor, subscription)
+                : new PersistentDispatcherMultipleConsumers(topic, cursor, subscription);
+    }
+
     @Test(timeOut = 30 * 1000)
     public void testTopicDeleteIfConsumerSetMismatchConsumerList() throws Exception {
         final String topicName = newTopicName();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
index 3a5c73315a01..498c000044ba 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java
@@ -939,7 +939,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
                 }
 
                 @Override
-                public synchronized void readMoreEntries() {
+                protected synchronized void internalReadMoreEntries() {
                     readMoreEntriesCalled.incrementAndGet();
                 }
 
@@ -960,7 +960,7 @@ public class PersistentStickyKeyDispatcherMultipleConsumersTest {
                 }
 
                 @Override
-                public synchronized void readMoreEntries() {
+                protected synchronized void internalReadMoreEntries() {
                     readMoreEntriesCalled.incrementAndGet();
                 }
 


(pulsar) 04/27: [fix][ml] Fix active cursor position tracking after untracking (#26629)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit b15171f82ded6ef05c0a16631b8156fa792d483d
Author: sinan liu <li...@gmail.com>
AuthorDate: Fri Sep 18 03:06:05 2026 +0800

    [fix][ml] Fix active cursor position tracking after untracking (#26629)
    
    (cherry picked from commit 467159eb8807e01dc6acb34daebeef44d98f8d14)
---
 .../impl/ActiveManagedCursorContainerImpl.java     |  14 +-
 ...ActiveManagedCursorContainerUntrackingTest.java | 163 +++++++++++++++++++++
 .../mledger/impl/ManagedLedgerBkTest.java          |  44 ++++++
 3 files changed, 218 insertions(+), 3 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerImpl.java
index 889dee2e2058..9d82d9716e84 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerImpl.java
@@ -387,8 +387,10 @@ public class ActiveManagedCursorContainerImpl implements ActiveManagedCursorCont
                 return;
             }
             if (newPosition == null) {
-                pendingRemovedCursors.put(cursor.getName(), node);
-                node.pendingRemove = true;
+                if (node.position != null) {
+                    pendingRemovedCursors.put(cursor.getName(), node);
+                    node.pendingRemove = true;
+                }
                 node.pendingPosition = null;
             } else {
                 if (node.pendingRemove) {
@@ -431,6 +433,7 @@ public class ActiveManagedCursorContainerImpl implements ActiveManagedCursorCont
         if (!pendingRemovedCursors.isEmpty()) {
             for (Node node : pendingRemovedCursors.values()) {
                 removeNodeFromList(node);
+                node.position = null;
                 node.pendingRemove = false;
             }
             pendingRemovedCursors.clear();
@@ -481,7 +484,12 @@ public class ActiveManagedCursorContainerImpl implements ActiveManagedCursorCont
                 node.position = node.pendingPosition;
                 node.pendingPosition = null;
             }
-            if (node.position != null && !node.pendingRemove) {
+            if (node.pendingRemove) {
+                // Untracked cursors stay in the map and may later be inserted incrementally.
+                node.position = null;
+                node.prev = null;
+                node.next = null;
+            } else if (node.position != null) {
                 activeNodes.add(node);
                 trackedNodeCount++;
             }
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerUntrackingTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerUntrackingTest.java
new file mode 100644
index 000000000000..0bea95482c55
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ActiveManagedCursorContainerUntrackingTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.apache.bookkeeper.mledger.impl.MockManagedCursor.createCursor;
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class ActiveManagedCursorContainerUntrackingTest {
+    private ActiveManagedCursorContainerImpl container;
+    private final List<ManagedCursor> cursors = new ArrayList<>();
+    private final List<Position> positions = new ArrayList<>();
+
+    @BeforeMethod(alwaysRun = true)
+    public void createContainer() {
+        container = new ActiveManagedCursorContainerImpl();
+        positions.clear();
+        cursors.clear();
+        for (int i = 0; i < 7; i++) {
+            positions.add(PositionFactory.create(1, i));
+        }
+        for (int i = 0; i < 5; i++) {
+            Position position = positions.get(i + 1);
+            ManagedCursor cursor = createCursor(container, "cursor" + i, position);
+            container.add(cursor, position);
+            cursors.add(cursor);
+        }
+        assertThat(container.size()).isEqualTo(5);
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+    }
+
+    @DataProvider
+    public Object[][] untrackingPaths() {
+        return new Object[][] {{false, false}, {false, true}, {true, false}, {true, true}};
+    }
+
+    @Test(dataProvider = "untrackingPaths")
+    public void testFlushedUntrackingRemainsUntracked(boolean rebuild, boolean useAdd) {
+        ManagedCursor cursor = cursors.get(2);
+        untrack(cursor, useAdd);
+        flushUntracking(rebuild);
+
+        assertUntracked(cursor);
+        // Updating two of the four remaining nodes forces a rebuild. It must not resurrect the cursor.
+        queueRebuild();
+        assertUntracked(cursor);
+        assertThat(container.getNumberOfCursorsAtSamePositionOrBefore(cursors.get(4))).isEqualTo(4);
+        container.checkOrderingAndNumberOfCursorsState();
+    }
+
+    @DataProvider
+    public Object[][] reactivationPaths() {
+        return new Object[][] {
+                {false, false, false}, {false, true, false}, {true, false, false}, {true, true, false},
+                {false, false, true}, {false, true, true}, {true, false, true}, {true, true, true}
+        };
+    }
+
+    @Test(dataProvider = "reactivationPaths", timeOut = 30000)
+    public void testRetrackAfterFlush(boolean rebuild, boolean moveForward, boolean useAdd) {
+        ManagedCursor cursor = cursors.get(2);
+        untrack(cursor, false);
+        flushUntracking(rebuild);
+        Position restored = positions.get(moveForward ? 6 : 0);
+        if (useAdd) {
+            container.add(cursor, restored);
+        } else {
+            container.updateCursor(cursor, restored);
+        }
+        // One update among four tracked nodes forces incremental insertion after either removal path.
+        assertThat(container.getNumberOfCursorsAtSamePositionOrBefore(cursor)).isEqualTo(moveForward ? 5 : 1);
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(moveForward ? positions.get(1) : restored);
+        for (int i = 0; i < cursors.size(); i++) {
+            if (i != 2) {
+                int rankWithoutRestored = i < 2 ? i + 1 : i;
+                assertThat(container.getNumberOfCursorsAtSamePositionOrBefore(cursors.get(i)))
+                        .as("rank of cursor%s", i).isEqualTo(rankWithoutRestored + (moveForward ? 0 : 1));
+            }
+        }
+        container.checkOrderingAndNumberOfCursorsState();
+    }
+
+    @Test
+    public void testRepeatedUntrackingAfterFlush() {
+        ManagedCursor cursor = cursors.get(2);
+        container.updateCursor(cursor, null);
+        container.getSlowestCursorPosition();
+        container.updateCursor(cursor, null);
+
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+        assertUntracked(cursor);
+        container.checkOrderingAndNumberOfCursorsState();
+    }
+
+    @Test
+    public void testUntrackingBeforeFirstPositionFlush() {
+        // The last cursor never enters this container's ordered list.
+        ActiveManagedCursorContainerImpl other = new ActiveManagedCursorContainerImpl();
+        for (int i = 0; i < 4; i++) {
+            other.add(cursors.get(i), positions.get(i + 1));
+        }
+        other.getSlowestCursorPosition();
+        ManagedCursor cursor = cursors.get(4);
+        other.add(cursor, null);
+        other.updateCursor(cursor, null);
+
+        assertThat(other.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+        assertThat(other.getNumberOfCursorsAtSamePositionOrBefore(cursor)).isZero();
+        assertThat(other.getNumberOfCursorsAtSamePositionOrBefore(cursors.get(3))).isEqualTo(4);
+        other.checkOrderingAndNumberOfCursorsState();
+    }
+
+    private void untrack(ManagedCursor cursor, boolean useAdd) {
+        // Normal broker deactivation removes the cursor. Null untracking is a separate container API operation.
+        if (useAdd) {
+            container.add(cursor, null);
+        } else {
+            container.updateCursor(cursor, null);
+        }
+    }
+
+    private void flushUntracking(boolean rebuild) {
+        if (rebuild) {
+            queueRebuild();
+        }
+        // No position updates selects incremental removal; two updates among five nodes select a rebuild.
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+    }
+
+    private void queueRebuild() {
+        container.updateCursor(cursors.get(0), positions.get(1));
+        container.updateCursor(cursors.get(4), positions.get(5));
+    }
+
+    private void assertUntracked(ManagedCursor cursor) {
+        assertThat(container.getNumberOfCursorsAtSamePositionOrBefore(cursor)).isZero();
+        assertThat(container.size()).isEqualTo(5);
+        assertThat(container.get(cursor.getName())).isSameAs(cursor);
+    }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index ee6a9c78658e..9740db647d90 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -19,6 +19,7 @@
 package org.apache.bookkeeper.mledger.impl;
 
 import static org.apache.pulsar.common.util.PortManager.releaseLockedPort;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertNotNull;
@@ -62,6 +63,7 @@ import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager;
 import org.apache.bookkeeper.mledger.proto.PositionInfo;
 import org.apache.bookkeeper.mledger.util.ThrowableToStringUtil;
 import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
 import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
 import org.apache.pulsar.common.util.FutureUtil;
 import org.awaitility.Awaitility;
@@ -102,6 +104,48 @@ public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
         entries.forEach(Entry::release);
     }
 
+    @Test
+    public void testSeekRetracksCursorAfterUntracking() throws Exception {
+        @Cleanup("shutdown")
+        ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+        ManagedLedgerConfig config = new ManagedLedgerConfig()
+                .setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1);
+        config.setPulsarMessageEntries(false);
+        config.setCacheEvictionByExpectedReadCount(true);
+        config.setCacheEvictionByMarkDeletedPosition(false);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("cursor-untracking-" + testName, config);
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i < 7; i++) {
+            positions.add(ledger.addEntry(("entry-" + i).getBytes(StandardCharsets.UTF_8)));
+        }
+        List<ManagedCursor> cursors = new ArrayList<>();
+        for (int i = 0; i < 5; i++) {
+            ManagedCursor cursor = ledger.openCursor("cursor" + i, InitialPosition.Earliest);
+            cursor.seek(positions.get(i + 1));
+            cursor.setActive();
+            cursors.add(cursor);
+        }
+        ActiveManagedCursorContainer container = ledger.getActiveCursors();
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+        ManagedCursor cursor = cursors.get(2);
+        // Null untracking is a container API operation; normal broker deactivation removes the cursor.
+        container.updateCursor(cursor, null);
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+
+        // Exercise ManagedCursorImpl -> ManagedLedgerImpl -> the active cursor container.
+        // One update among four tracked nodes forces incremental reinsertion.
+        cursor.seek(positions.get(6));
+        assertThat(ledger.getNumberOfCursorsAtSamePositionOrBefore(cursor)).isEqualTo(5);
+        assertThat(container.getSlowestCursorPosition()).isEqualTo(positions.get(1));
+        assertThat(container.size()).isEqualTo(5);
+        for (int i = 0; i < cursors.size(); i++) {
+            if (i != 2) {
+                assertThat(ledger.getNumberOfCursorsAtSamePositionOrBefore(cursors.get(i)))
+                        .as("rank of cursor%s", i).isEqualTo(i < 2 ? i + 1 : i);
+            }
+        }
+    }
+
     @Test
     public void testBookieFailure() throws Exception {
         ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);


(pulsar) 19/27: [fix][broker] Keep accepting Avro named type references written as objects (#26586)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 9e3cc4f1fa869067d560d341c6991eeaecba2eaa
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Sep 15 23:24:46 2026 +0300

    [fix][broker] Keep accepting Avro named type references written as objects (#26586)
    
    (cherry picked from commit 30fe4106e6266128cce74d333931e141a3f1e87c)
---
 .../schema/AvroSchemaBasedCompatibilityCheck.java  |  10 +-
 .../schema/JsonSchemaCompatibilityCheck.java       |   6 +-
 .../service/schema/SchemaRegistryServiceImpl.java  |   9 +-
 .../validator/StructSchemaDataValidator.java       |  19 ++-
 .../schema/BaseAvroSchemaCompatibilityTest.java    |  20 +++
 .../schema/validator/SchemaDataValidatorTest.java  |  33 +++++
 .../impl/schema/generic/GenericJsonRecord.java     |   3 +-
 .../pulsar/client/impl/schema/util/SchemaUtil.java |   3 +-
 .../client/impl/schema/util/SchemaUtilTest.java    |  19 +++
 .../pulsar/common/schema/AvroSchemaCompat.java     | 147 +++++++++++++++++++++
 .../pulsar/common/schema/AvroSchemaCompatTest.java | 101 ++++++++++++++
 11 files changed, 348 insertions(+), 22 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java
index 56012a3ddd7d..0fb94c4e3d4f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java
@@ -52,14 +52,10 @@ abstract class AvroSchemaBasedCompatibilityCheck implements SchemaCompatibilityC
         checkArgument(from != null, "check compatibility list is null");
         try {
             for (SchemaData schemaData : from) {
-                Schema.Parser parser =
-                        new Schema.Parser(StructSchemaDataValidator.COMPATIBLE_NAME_VALIDATOR);
-                parser.setValidateDefaults(false);
-                fromList.addFirst(parser.parse(new String(schemaData.getData(), UTF_8)));
+                fromList.addFirst(StructSchemaDataValidator.parseAvroSchema(
+                        new String(schemaData.getData(), UTF_8), false));
             }
-            Schema.Parser parser = new Schema.Parser(StructSchemaDataValidator.COMPATIBLE_NAME_VALIDATOR);
-            parser.setValidateDefaults(false);
-            Schema toSchema = parser.parse(new String(to.getData(), UTF_8));
+            Schema toSchema = StructSchemaDataValidator.parseAvroSchema(new String(to.getData(), UTF_8), false);
             SchemaValidator schemaValidator = createSchemaValidator(strategy);
             schemaValidator.validate(toSchema, fromList);
         } catch (SchemaParseException e) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
index dc8be3e0651a..4811277d8a30 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java
@@ -22,7 +22,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
 import java.io.IOException;
-import org.apache.avro.Schema;
 import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException;
 import org.apache.pulsar.broker.service.schema.validator.StructSchemaDataValidator;
 import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy;
@@ -106,10 +105,7 @@ public class JsonSchemaCompatibilityCheck extends AvroSchemaBasedCompatibilityCh
 
     private boolean isAvroSchema(SchemaData schemaData) {
         try {
-
-            Schema.Parser fromParser = new Schema.Parser(StructSchemaDataValidator.COMPATIBLE_NAME_VALIDATOR);
-            fromParser.setValidateDefaults(false);
-            Schema fromSchema = fromParser.parse(new String(schemaData.getData(), UTF_8));
+            StructSchemaDataValidator.parseAvroSchema(new String(schemaData.getData(), UTF_8), false);
             return true;
         } catch (Exception e) {
             return false;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
index a5b6bd884088..4be7ab2b04a8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java
@@ -414,14 +414,13 @@ public class SchemaRegistryServiceImpl implements SchemaRegistryService {
         final CompletableFuture<SchemaVersion> completableFuture = new CompletableFuture<>();
         SchemaVersion schemaVersion;
         if (isUsingAvroSchemaParser(schemaData.getType())) {
-            Schema.Parser parser = new Schema.Parser(StructSchemaDataValidator.COMPATIBLE_NAME_VALIDATOR);
-            Schema newSchema = parser.parse(new String(schemaData.getData(), UTF_8));
+            Schema newSchema = StructSchemaDataValidator.parseAvroSchema(
+                    new String(schemaData.getData(), UTF_8), true);
 
             for (SchemaAndMetadata schemaAndMetadata : schemaAndMetadataList) {
                 if (isUsingAvroSchemaParser(schemaAndMetadata.schema.getType())) {
-                    Schema.Parser existParser =
-                            new Schema.Parser(StructSchemaDataValidator.COMPATIBLE_NAME_VALIDATOR);
-                    Schema existSchema = existParser.parse(new String(schemaAndMetadata.schema.getData(), UTF_8));
+                    Schema existSchema = StructSchemaDataValidator.parseAvroSchema(
+                            new String(schemaAndMetadata.schema.getData(), UTF_8), true);
                     if (newSchema.equals(existSchema) && schemaAndMetadata.schema.getType() == schemaData.getType()) {
                         schemaVersion = schemaAndMetadata.version;
                         completableFuture.complete(schemaVersion);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java
index 7417e06c2d44..d47074d16168 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java
@@ -26,6 +26,7 @@ import org.apache.avro.NameValidator;
 import org.apache.avro.Schema;
 import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException;
 import org.apache.pulsar.common.protocol.schema.SchemaData;
+import org.apache.pulsar.common.schema.AvroSchemaCompat;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 
@@ -61,9 +62,7 @@ public class StructSchemaDataValidator implements SchemaDataValidator {
         byte[] data = schemaData.getData();
 
         try {
-            Schema.Parser avroSchemaParser = new Schema.Parser(COMPATIBLE_NAME_VALIDATOR);
-            avroSchemaParser.setValidateDefaults(false);
-            Schema schema = avroSchemaParser.parse(new String(data, UTF_8));
+            Schema schema = parseAvroSchema(new String(data, UTF_8), false);
             if (SchemaType.AVRO.equals(schemaData.getType())) {
                 checkAvroSchemaTypeSupported(schema);
             }
@@ -86,6 +85,20 @@ public class StructSchemaDataValidator implements SchemaDataValidator {
         }
     }
 
+    /**
+     * Parse an Avro schema definition the way the broker accepts it: with the compatible name validator and
+     * with named type references in the pre Avro 1.12.2 object form.
+     *
+     * @param schemaDefinition the schema definition as JSON
+     * @param validateDefaults whether to validate the default values of the schema
+     * @return the parsed schema
+     */
+    public static Schema parseAvroSchema(String schemaDefinition, boolean validateDefaults) {
+        Schema.Parser parser = new Schema.Parser(COMPATIBLE_NAME_VALIDATOR);
+        parser.setValidateDefaults(validateDefaults);
+        return parser.parse(AvroSchemaCompat.normalizeNamedTypeReferences(schemaDefinition));
+    }
+
     static void checkAvroSchemaTypeSupported(Schema schema) throws InvalidSchemaDataException {
             switch (schema.getType()) {
                 case RECORD: {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java
index cbb59c2ecde9..5fff51cbcd7f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java
@@ -152,6 +152,26 @@ public abstract class BaseAvroSchemaCompatibilityTest {
      * Make sure the new schema is forward- and
      * backward-compatible from the latest to newest and from the newest to latest.
      */
+    @Test
+    public void testLegacyNamedTypeReferenceForm() {
+        // Avro 1.12.2 rejects named type references written as {"type": "name"} (AVRO-4176); schemas stored in
+        // that form must remain compatible with the same schema written with bare names
+        String colorEnum = "{\"type\":\"enum\",\"name\":\"Color\",\"namespace\":\"org.example.shapes\","
+                + "\"symbols\":[\"RED\",\"BLUE\"]}";
+        String legacyForm = "{\"type\":\"record\",\"name\":\"Drawing\",\"namespace\":\"org.example.shapes\","
+                + "\"fields\":[{\"name\":\"background\",\"type\":" + colorEnum + "},"
+                + "{\"name\":\"outline\",\"type\":{\"type\":\"org.example.shapes.Color\"}}]}";
+        String bareNameForm = legacyForm.replace("{\"type\":\"org.example.shapes.Color\"}",
+                "\"org.example.shapes.Color\"");
+        SchemaCompatibilityCheck schemaCompatibilityCheck = getSchemaCheck();
+        Assert.assertTrue(schemaCompatibilityCheck.isCompatible(getSchemaData(legacyForm),
+                        getSchemaData(bareNameForm), SchemaCompatibilityStrategy.FULL),
+                "the legacy and the bare name form of a schema are the same schema");
+        Assert.assertTrue(schemaCompatibilityCheck.isCompatible(getSchemaData(bareNameForm),
+                        getSchemaData(legacyForm), SchemaCompatibilityStrategy.FULL),
+                "the legacy and the bare name form of a schema are the same schema");
+    }
+
     @Test
     public void testFullCompatibility() {
         SchemaCompatibilityCheck schemaCompatibilityCheck = getSchemaCheck();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
index 079405407f4b..ec610d321540 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaDataValidatorTest.java
@@ -367,4 +367,37 @@ public class SchemaDataValidatorTest {
         StructSchemaDataValidator.of().validate(SchemaData.fromSchemaInfo(protobufSchema.getSchemaInfo()));
     }
 
+    // Avro 1.12.2 rejects named type references written as {"type": "name"} (AVRO-4176); such schemas were
+    // accepted before and must still be accepted
+
+    @DataProvider(name = "legacyNamedTypeReferenceSchemas")
+    public Object[][] legacyNamedTypeReferenceSchemas() {
+        String colorEnum = "{\"type\":\"enum\",\"name\":\"Color\",\"namespace\":\"org.example.shapes\","
+                + "\"symbols\":[\"RED\",\"BLUE\"]}";
+        String drawing = "{\"type\":\"record\",\"name\":\"Drawing\",\"namespace\":\"org.example.shapes\","
+                + "\"fields\":[{\"name\":\"background\",\"type\":" + colorEnum + "},"
+                + "{\"name\":\"outline\",\"type\":{\"type\":\"org.example.shapes.Color\"}},"
+                + "{\"name\":\"palette\",\"type\":{\"type\":\"array\","
+                + "\"items\":{\"type\":\"org.example.shapes.Color\"}}}]}";
+        return new Object[][] {
+            { SchemaType.AVRO, drawing },
+            { SchemaType.JSON, drawing },
+        };
+    }
+
+    @Test(dataProvider = "legacyNamedTypeReferenceSchemas")
+    public void testLegacyNamedTypeReferenceIsAccepted(SchemaType type, String schemaDefinition)
+            throws InvalidSchemaDataException {
+        SchemaData data = SchemaData.builder()
+            .type(type)
+            .data(schemaDefinition.getBytes(UTF_8))
+            .build();
+        SchemaDataValidator.validateSchemaData(data, false);
+
+        org.apache.avro.Schema parsed = StructSchemaDataValidator.parseAvroSchema(schemaDefinition, false);
+        Assert.assertEquals(parsed.getField("outline").schema().getFullName(), "org.example.shapes.Color");
+        Assert.assertEquals(parsed.getField("palette").schema().getElementType().getFullName(),
+                "org.example.shapes.Color");
+    }
+
 }
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonRecord.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonRecord.java
index b0b919c3d1ee..52b82a7d45cb 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonRecord.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericJsonRecord.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pulsar.client.api.schema.Field;
+import org.apache.pulsar.common.schema.AvroSchemaCompat;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaType;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
@@ -130,7 +131,7 @@ public class GenericJsonRecord extends VersionedGenericRecord {
     private static org.apache.avro.Schema parseAvroSchema(String schemaJson) {
         final org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser();
         parser.setValidateDefaults(false);
-        return parser.parse(schemaJson);
+        return parser.parse(AvroSchemaCompat.normalizeNamedTypeReferences(schemaJson));
     }
 
     @Override
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
index 99fe54ae7b5b..8c246c2d0386 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/util/SchemaUtil.java
@@ -28,6 +28,7 @@ import org.apache.pulsar.client.api.schema.SchemaDefinition;
 import org.apache.pulsar.client.impl.schema.AvroSchema;
 import org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl;
 import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
+import org.apache.pulsar.common.schema.AvroSchemaCompat;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaType;
 
@@ -53,7 +54,7 @@ public class SchemaUtil {
     public static Schema parseAvroSchema(String schemaJson) {
         final Schema.Parser parser = new Schema.Parser(NameValidator.NO_VALIDATION);
         parser.setValidateDefaults(false);
-        return parser.parse(schemaJson);
+        return parser.parse(AvroSchemaCompat.normalizeNamedTypeReferences(schemaJson));
     }
 
     public static <T> SchemaInfo parseSchemaInfo(SchemaDefinition<T> schemaDefinition, SchemaType schemaType) {
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/util/SchemaUtilTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/util/SchemaUtilTest.java
index d80a25873a8e..c8f7a0f98a17 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/util/SchemaUtilTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/util/SchemaUtilTest.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.client.impl.schema.util;
 
 import static org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl.JSR310_CONVERSION_ENABLED;
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 import java.util.HashMap;
@@ -108,4 +109,22 @@ public class SchemaUtilTest {
                 .build();
     }
 
+    @Test
+    public void testParseAvroSchemaWithLegacyNamedTypeReference() {
+        // Avro 1.12.2 rejects named type references written as {"type": "name"} (AVRO-4176); schemas stored in
+        // that form must still be readable
+        String colorEnum = "{\"type\":\"enum\",\"name\":\"Color\",\"namespace\":\"org.example.shapes\","
+                + "\"symbols\":[\"RED\",\"BLUE\"]}";
+        String legacyForm = "{\"type\":\"record\",\"name\":\"Drawing\",\"namespace\":\"org.example.shapes\","
+                + "\"fields\":[{\"name\":\"background\",\"type\":" + colorEnum + "},"
+                + "{\"name\":\"outline\",\"type\":{\"type\":\"org.example.shapes.Color\"}},"
+                + "{\"name\":\"highlight\",\"type\":[\"null\",{\"type\":\"org.example.shapes.Color\"}],"
+                + "\"default\":null}]}";
+        String bareNameForm = legacyForm.replace("{\"type\":\"org.example.shapes.Color\"}",
+                "\"org.example.shapes.Color\"");
+
+        org.apache.avro.Schema parsed = SchemaUtil.parseAvroSchema(legacyForm);
+        assertEquals(parsed, SchemaUtil.parseAvroSchema(bareNameForm));
+        assertEquals(parsed.getField("outline").schema().getFullName(), "org.example.shapes.Color");
+    }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/schema/AvroSchemaCompat.java b/pulsar-common/src/main/java/org/apache/pulsar/common/schema/AvroSchemaCompat.java
new file mode 100644
index 000000000000..9e0dfa05fe75
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/schema/AvroSchemaCompat.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.schema;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.util.Set;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+
+/**
+ * Compatibility handling for Avro schema definitions that were written for Avro versions before 1.12.2.
+ *
+ * <p>Up to Avro 1.12.1 the parser accepted a reference to a previously defined named type written as a JSON
+ * object whose {@code type} attribute is the name of the type, for example {@code {"type": "org.example.Color"}},
+ * and resolved it to the named type while ignoring any other attribute of the object. Avro 1.12.2 rejects this
+ * form (AVRO-4176) and only accepts the bare name {@code "org.example.Color"}. Schema definitions in this form
+ * have been accepted and stored by the schema registry, and clients keep producing them, so they have to be
+ * rewritten to the bare name before they are passed to the Avro parser.
+ */
+public final class AvroSchemaCompat {
+
+    /**
+     * The values of {@code type} for which an object defines a schema. For any other value the object is a
+     * reference to a named type.
+     */
+    private static final Set<String> TYPE_KEYWORDS = Set.of(
+            "null", "boolean", "int", "long", "float", "double", "bytes", "string",
+            "record", "error", "enum", "array", "map", "fixed");
+
+    private AvroSchemaCompat() {
+    }
+
+    /**
+     * Rewrite named type references written as {@code {"type": "name"}} objects to the bare {@code "name"}.
+     *
+     * <p>Only positions that hold a schema are visited: the schema itself, field types, array items, map values
+     * and union members. Other JSON values, for example field defaults, are never modified.
+     *
+     * @param schemaDefinition the Avro schema definition as JSON
+     * @return the rewritten definition, or the given string unchanged if it contains no such reference or is not
+     *         valid JSON
+     */
+    public static String normalizeNamedTypeReferences(String schemaDefinition) {
+        if (schemaDefinition == null || schemaDefinition.indexOf('{') < 0) {
+            return schemaDefinition;
+        }
+        JsonNode root;
+        try {
+            root = ObjectMapperFactory.getMapper().reader().readTree(schemaDefinition);
+        } catch (JsonProcessingException e) {
+            // let the Avro parser report the error
+            return schemaDefinition;
+        }
+        NamedTypeReferenceRewriter rewriter = new NamedTypeReferenceRewriter();
+        JsonNode rewritten = rewriter.rewrite(root);
+        if (!rewriter.changed) {
+            return schemaDefinition;
+        }
+        try {
+            return ObjectMapperFactory.getMapper().writer().writeValueAsString(rewritten);
+        } catch (JsonProcessingException e) {
+            return schemaDefinition;
+        }
+    }
+
+    private static final class NamedTypeReferenceRewriter {
+
+        private boolean changed;
+
+        JsonNode rewrite(JsonNode node) {
+            if (node instanceof ArrayNode) {
+                ArrayNode union = (ArrayNode) node;
+                for (int i = 0; i < union.size(); i++) {
+                    union.set(i, rewrite(union.get(i)));
+                }
+                return union;
+            }
+            if (!(node instanceof ObjectNode)) {
+                // a bare name, or something invalid that the Avro parser reports
+                return node;
+            }
+            ObjectNode object = (ObjectNode) node;
+            JsonNode type = object.get("type");
+            if (type == null || !type.isTextual()) {
+                return object;
+            }
+            String typeName = type.textValue();
+            if (!TYPE_KEYWORDS.contains(typeName)) {
+                changed = true;
+                return TextNode.valueOf(typeName);
+            }
+            switch (typeName) {
+                case "record":
+                case "error":
+                    rewriteFieldTypes(object.get("fields"));
+                    break;
+                case "array":
+                    rewriteChild(object, "items");
+                    break;
+                case "map":
+                    rewriteChild(object, "values");
+                    break;
+                default:
+                    // primitive, enum and fixed types have no nested schema
+                    break;
+            }
+            return object;
+        }
+
+        private void rewriteFieldTypes(JsonNode fields) {
+            if (fields == null || !fields.isArray()) {
+                return;
+            }
+            for (JsonNode field : fields) {
+                if (field instanceof ObjectNode) {
+                    rewriteChild((ObjectNode) field, "type");
+                }
+            }
+        }
+
+        private void rewriteChild(ObjectNode object, String name) {
+            JsonNode child = object.get(name);
+            if (child != null) {
+                object.set(name, rewrite(child));
+            }
+        }
+    }
+}
diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/schema/AvroSchemaCompatTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/schema/AvroSchemaCompatTest.java
new file mode 100644
index 000000000000..d64745ab1541
--- /dev/null
+++ b/pulsar-common/src/test/java/org/apache/pulsar/common/schema/AvroSchemaCompatTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.schema;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertSame;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.testng.annotations.Test;
+
+public class AvroSchemaCompatTest {
+
+    private static final String COLOR_ENUM = "{\"type\":\"enum\",\"name\":\"Color\","
+            + "\"namespace\":\"org.example.shapes\",\"symbols\":[\"RED\",\"BLUE\"]}";
+
+    @Test
+    public void testNamedTypeReferencesAreRewrittenInAllSchemaPositions() throws Exception {
+        String legacy = "{\"type\":\"record\",\"name\":\"Drawing\",\"namespace\":\"org.example.shapes\",\"fields\":["
+                + "{\"name\":\"background\",\"type\":" + COLOR_ENUM + "},"
+                + "{\"name\":\"outline\",\"type\":{\"type\":\"org.example.shapes.Color\"}},"
+                + "{\"name\":\"palette\",\"type\":{\"type\":\"array\","
+                + "\"items\":{\"type\":\"org.example.shapes.Color\"}}},"
+                + "{\"name\":\"labels\",\"type\":{\"type\":\"map\","
+                + "\"values\":{\"type\":\"org.example.shapes.Color\"}}},"
+                + "{\"name\":\"highlight\",\"type\":[\"null\",{\"type\":\"org.example.shapes.Color\"}],"
+                + "\"default\":null}"
+                + "]}";
+        String expected = legacy.replace("{\"type\":\"org.example.shapes.Color\"}", "\"org.example.shapes.Color\"");
+
+        assertJsonEquals(AvroSchemaCompat.normalizeNamedTypeReferences(legacy), expected);
+    }
+
+    @Test
+    public void testNestedRecordsAreVisited() throws Exception {
+        String legacy = "{\"type\":\"record\",\"name\":\"Canvas\",\"namespace\":\"org.example.shapes\",\"fields\":["
+                + "{\"name\":\"colors\",\"type\":{\"type\":\"array\",\"items\":" + COLOR_ENUM + "}},"
+                + "{\"name\":\"frame\",\"type\":{\"type\":\"record\",\"name\":\"Frame\",\"fields\":["
+                + "{\"name\":\"color\",\"type\":{\"type\":\"org.example.shapes.Color\"}}]}}"
+                + "]}";
+        String expected = legacy.replace("{\"type\":\"org.example.shapes.Color\"}", "\"org.example.shapes.Color\"");
+
+        assertJsonEquals(AvroSchemaCompat.normalizeNamedTypeReferences(legacy), expected);
+    }
+
+    @Test
+    public void testTopLevelNamedTypeReferenceIsRewritten() {
+        assertEquals(AvroSchemaCompat.normalizeNamedTypeReferences("{\"type\":\"org.example.shapes.Color\"}"),
+                "\"org.example.shapes.Color\"");
+    }
+
+    @Test
+    public void testDefaultValuesAreNotModified() throws Exception {
+        // the default of "metadata" is a record value that happens to have a field named "type"
+        String schema = "{\"type\":\"record\",\"name\":\"Document\",\"fields\":["
+                + "{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":["
+                + "{\"name\":\"type\",\"type\":\"string\"}]},\"default\":{\"type\":\"org.example.NotASchema\"}},"
+                + "{\"name\":\"previous\",\"type\":{\"type\":\"Metadata\"},\"default\":{\"type\":\"draft\"}}"
+                + "]}";
+        String expected = schema.replace("{\"type\":\"Metadata\"}", "\"Metadata\"");
+
+        assertJsonEquals(AvroSchemaCompat.normalizeNamedTypeReferences(schema), expected);
+    }
+
+    @Test
+    public void testSchemaWithoutLegacyReferencesIsReturnedUnchanged() {
+        String schema = "{\"type\":\"record\",\"name\":\"Point\",\"fields\":["
+                + "{\"name\":\"x\",\"type\":\"double\"},{\"name\":\"y\",\"type\":{\"type\":\"double\"}},"
+                + "{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"string\"}}]}";
+        assertSame(AvroSchemaCompat.normalizeNamedTypeReferences(schema), schema);
+        assertSame(AvroSchemaCompat.normalizeNamedTypeReferences("\"string\""), "\"string\"");
+    }
+
+    @Test
+    public void testInvalidJsonIsReturnedUnchanged() {
+        String notJson = "{\"type\":\"record\",";
+        assertSame(AvroSchemaCompat.normalizeNamedTypeReferences(notJson), notJson);
+        assertSame(AvroSchemaCompat.normalizeNamedTypeReferences(null), null);
+    }
+
+    private static void assertJsonEquals(String actual, String expected) throws Exception {
+        JsonNode actualNode = ObjectMapperFactory.getMapper().reader().readTree(actual);
+        JsonNode expectedNode = ObjectMapperFactory.getMapper().reader().readTree(expected);
+        assertEquals(actualNode, expectedNode);
+    }
+}


(pulsar) 09/27: [fix] Fix ManagedLedgerImpl's pendingAddEntries leak. (#25240)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit eaaa8e8428472e29efa4552562e88f8e92b552de
Author: zzb <48...@users.noreply.github.com>
AuthorDate: Thu May 14 20:53:51 2026 +0800

    [fix] Fix ManagedLedgerImpl's pendingAddEntries leak. (#25240)
    
    Co-authored-by: zhaizhibo <zh...@kuaishou.com>
    (cherry picked from commit de56ab517f2762f3315e6f207f7f60e363580b9e)
---
 .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 67 +++++++++++-----
 .../bookkeeper/mledger/impl/ManagedLedgerTest.java | 88 ++++++++++++++++++++++
 .../broker/service/persistent/PersistentTopic.java |  6 +-
 3 files changed, 142 insertions(+), 19 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index f91e0baf37f0..4c52d640c547 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1685,6 +1685,7 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
 
         factory.close(this);
         STATE_UPDATER.set(this, State.Closed);
+        clearPendingAddEntries(new ManagedLedgerAlreadyClosedException("Managed ledger is closed"));
         cancelScheduledTasks();
 
         LedgerHandle lh = currentLedger;
@@ -1785,19 +1786,37 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
                     }
                     ledgersStat = stat;
                     synchronized (ManagedLedgerImpl.this) {
-                        LedgerHandle originalCurrentLedger = currentLedger;
-                        ledgers.put(lh.getId(), newLedger);
-                        currentLedger = lh;
-                        currentLedgerTimeoutTriggered = new AtomicBoolean();
-                        currentLedgerEntries = 0;
-                        currentLedgerSize = 0;
-                        updateLedgersIdsComplete(originalCurrentLedger);
-                        mbean.addLedgerSwitchLatencySample(System.currentTimeMillis()
-                                - lastLedgerCreationInitiationTimestamp, TimeUnit.MILLISECONDS);
-                        // May need to update the cursor position
-                        maybeUpdateCursorBeforeTrimmingConsumedLedger();
+                        try {
+                            State state = STATE_UPDATER.get(ManagedLedgerImpl.this);
+                            if (state == State.Closed || state.isFenced()) {
+                                if (log.isDebugEnabled()) {
+                                    log.debug("[{}] Skip ledger update after create complete, ledger is closed"
+                                            + " or fenced", name);
+                                }
+                                lh.closeAsync().exceptionally(e -> {
+                                    if (e != null) {
+                                        log.error("[{}] Failed to close ledger {}: {}", name, lh.getId(),
+                                                e.getMessage());
+                                    }
+                                    return null;
+                                });
+                            } else {
+                                LedgerHandle originalCurrentLedger = currentLedger;
+                                ledgers.put(lh.getId(), newLedger);
+                                currentLedger = lh;
+                                currentLedgerTimeoutTriggered = new AtomicBoolean();
+                                currentLedgerEntries = 0;
+                                currentLedgerSize = 0;
+                                updateLedgersIdsComplete(originalCurrentLedger);
+                                mbean.addLedgerSwitchLatencySample(System.currentTimeMillis()
+                                        - lastLedgerCreationInitiationTimestamp, TimeUnit.MILLISECONDS);
+                                // May need to update the cursor position
+                                maybeUpdateCursorBeforeTrimmingConsumedLedger();
+                            }
+                        } finally {
+                            metadataMutex.unlock();
+                        }
                     }
-                    metadataMutex.unlock();
                 }
 
                 @Override
@@ -1978,6 +1997,9 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
             // The managed ledger was closed during the write operation
             clearPendingAddEntries(new ManagedLedgerAlreadyClosedException("Managed ledger was already closed"));
             return;
+        } else if (state.isFenced()) {
+            clearPendingAddEntries(new ManagedLedgerFencedException("Managed ledger is fenced"));
+            return;
         } else {
             // In case we get multiple write errors for different outstanding write request, we should close the ledger
             // just once
@@ -2110,9 +2132,9 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
         return managedLedgerInterceptor;
     }
 
-    void clearPendingAddEntries(ManagedLedgerException e) {
-        while (!pendingAddEntries.isEmpty()) {
-            OpAddEntry op = pendingAddEntries.poll();
+    synchronized void clearPendingAddEntries(ManagedLedgerException e) {
+        OpAddEntry op;
+        while ((op = pendingAddEntries.poll()) != null) {
             op.failed(e);
         }
     }
@@ -4449,7 +4471,8 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
         return activeCursors.get(cursor.getName()) != null;
     }
 
-    private boolean currentLedgerIsFull() {
+    @VisibleForTesting
+    protected boolean currentLedgerIsFull() {
         if (!factory.isMetadataServiceAvailable()) {
             // We don't want to trigger metadata operations if we already know that we're currently disconnected
             return false;
@@ -4540,12 +4563,20 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
     @VisibleForTesting
     public synchronized void setFenced() {
         log.info("{} Moving to Fenced state", name);
-        STATE_UPDATER.set(this, State.Fenced);
+        State prev = STATE_UPDATER.getAndSet(this, State.Fenced);
+        if (prev != State.Fenced) {
+            clearPendingAddEntries(new ManagedLedgerFencedException("ManagedLedger "
+                + name + " is fenced"));
+        }
     }
 
     synchronized void setFencedForDeletion() {
         log.info("{} Moving to FencedForDeletion state", name);
-        STATE_UPDATER.set(this, State.FencedForDeletion);
+        State prev = STATE_UPDATER.getAndSet(this, State.FencedForDeletion);
+        if (prev != State.FencedForDeletion) {
+            clearPendingAddEntries(new ManagedLedgerFencedException("ManagedLedger "
+                + name + " is fenced"));
+        }
     }
 
     MetaStore getStore() {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
index c1efa2a57458..e39bc66d10b2 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
@@ -786,6 +786,94 @@ public class ManagedLedgerTest extends MockedBookKeeperTestCase {
         fail("Should have thrown an exception in the above line");
     }
 
+    @Test(timeOut = 30000)
+    public void testCloseManagedLedgerAfterRollover() throws Exception {
+        ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
+        config.setMaxCacheSize(0);
+        @Cleanup("shutdown")
+        ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, config);
+        ManagedLedgerImpl realLedger = (ManagedLedgerImpl) factory.open("my_test_ledger");
+        ManagedLedgerImpl ledger = Mockito.spy(realLedger);
+        AtomicBoolean onlyOnce = new AtomicBoolean(false);
+        when(ledger.currentLedgerIsFull()).thenAnswer(invocation -> onlyOnce.compareAndSet(false, true));
+        OpAddEntry realOp = OpAddEntry.createNoRetainBuffer(ledger,
+                    ByteBufAllocator.DEFAULT.buffer(128), null, null, new AtomicBoolean());
+        OpAddEntry op = spy(realOp);
+        CountDownLatch createLatch = new CountDownLatch(1);
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        doAnswer(invocationOnMock -> {
+            // Simulate that before the rollover is completed, new write requests arrive,
+            // and after these write requests are added to pendingAddEntries, the ledger is closed.
+            log.info("before add, ledger state: " + ledger.getState());
+            for (int i = 0; i < 10; ++i) {
+                ledger.internalAsyncAddEntry(OpAddEntry.createNoRetainBuffer(ledger,
+                        ByteBufAllocator.DEFAULT.buffer(128), null, null, new AtomicBoolean()));
+            }
+            ledger.asyncClose(new CloseCallback() {
+                @Override
+                public void closeComplete(Object ctx) {
+                    log.info("closeComplete finished, ledger state: " + ledger.getState());
+                    closeLatch.countDown();
+                }
+
+                @Override
+                public void closeFailed(ManagedLedgerException exception, Object ctx) {
+                    log.info("closeFailed, ex: " + exception.getMessage() + ", state: " + ledger.getState());
+                    closeLatch.countDown();
+                }
+            }, null);
+            log.info("after add, ledger state: " + ledger.getState());
+            return invocationOnMock.callRealMethod();
+        }).when(ledger).asyncCreateLedger(any(), any(), any(), any(), any());
+        doAnswer(invocationOnMock -> {
+            Object o = invocationOnMock.callRealMethod();
+            log.info("createComplete finished, state: " + ledger.getState());
+            ledger.executor.execute(createLatch::countDown);
+            return o;
+        }).when(ledger).createComplete(anyInt(), any(), any());
+        ledger.internalAsyncAddEntry(op);
+        createLatch.await();
+        closeLatch.await();
+        Assert.assertEquals(ledger.pendingAddEntries.size(), 0);
+    }
+
+    @Test(timeOut = 20000)
+    public void testFencedManagedLedgerAfterAdd() throws Exception {
+        @Cleanup("shutdown")
+        ManagedLedgerFactory factory1 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+        ManagedLedgerImpl realLedger = (ManagedLedgerImpl) factory1.open("my_test_ledger");
+        ManagedLedgerImpl ledger = spy(realLedger);
+
+        int sendNum = 10;
+        CountDownLatch sendLatch = new CountDownLatch(sendNum);
+        CountDownLatch fencedLatch = new CountDownLatch(1);
+        doAnswer(invocationOnMock -> {
+            stopBookKeeper();
+            stopMetadataStore();
+            for (int i = 0; i < sendNum; ++i) {
+                ledger.internalAsyncAddEntry(OpAddEntry.createNoRetainBuffer(ledger,
+                    ByteBufAllocator.DEFAULT.buffer(128), new AddEntryCallback() {
+                    @Override
+                    public void addComplete(Position position, ByteBuf entryData, Object ctx) {
+                        sendLatch.countDown();
+                    }
+
+                    @Override
+                    public void addFailed(ManagedLedgerException exception, Object ctx) {
+                        sendLatch.countDown();
+                    }
+                }, null, new AtomicBoolean()));
+            }
+            Object o = invocationOnMock.callRealMethod();
+            fencedLatch.countDown();
+            return o;
+        }).when(ledger).setFenced();
+        ledger.setFenced();
+        fencedLatch.await();
+        sendLatch.await();
+        assertEquals(ledger.pendingAddEntries.size(), 0);
+    }
+
     @Test(timeOut = 20000)
     public void deleteAndReopen() throws Exception {
         ManagedLedger ledger = factory.open("my_test_ledger");
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index c17a6d209447..df307e651e06 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -772,18 +772,22 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
          Instead, we will rely on the service unit state channel's bundle(topic) transfer protocol.
          At the end of the transfer protocol, at Owned state, the source broker should close the topic properly.
          */
+        PublishContext callback = (PublishContext) ctx;
         if (transferring) {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] Failed to persist msg in store: {} while transferring.",
                         topic, exception.getMessage(), exception);
             }
+            callback.completed(new TopicClosedException(exception), -1, -1);
+            decrementPendingWriteOpsAndCheck();
             return;
         }
 
-        PublishContext callback = (PublishContext) ctx;
         if (exception instanceof ManagedLedgerFencedException) {
             // If the managed ledger has been fenced, we cannot continue using it. We need to close and reopen
             close();
+            callback.completed(new TopicFencedException(exception.getMessage()), -1, -1);
+            decrementPendingWriteOpsAndCheck();
         } else {
             // fence topic when failed to write a message to BK
             fence();


(pulsar) 26/27: [fix][broker] Check topic permissions for binary GetSchema and GetOrCreateSchema (#26643)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 320c5fcc487904528d55b182d86fe449b89f440f
Author: kannar <ka...@gmail.com>
AuthorDate: Sat Sep 19 17:09:36 2026 +0200

    [fix][broker] Check topic permissions for binary GetSchema and GetOrCreateSchema (#26643)
    
    Check topic permissions for binary GetSchema and GetOrCreateSchema requests and retain the schema and transaction tests when merging master.
    
    (cherry picked from commit 0f8af3259ec8b233374b42bccd79338c397a40a3)
---
 .../apache/pulsar/broker/service/ServerCnx.java    | 63 ++++++++++----
 .../pulsar/broker/service/ServerCnxTest.java       | 95 ++++++++++++++++++++++
 .../broker/service/utils/ClientChannelHelper.java  | 12 +++
 3 files changed, 155 insertions(+), 15 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 185b2ad2b5ba..b1c34d14650f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -2779,26 +2779,40 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
         }
 
         final String topic = commandGetSchema.getTopic();
-        String schemaName;
+        final TopicName topicName;
+        final String schemaName;
         try {
-            schemaName = TopicName.get(topic).getSchemaName();
+            topicName = TopicName.get(topic);
+            schemaName = topicName.getSchemaName();
         } catch (Throwable t) {
             commandSender.sendGetSchemaErrorResponse(requestId, ServerError.InvalidTopicName, t.getMessage());
             return;
         }
+        final SchemaVersion requestedVersion = schemaVersion;
 
-        schemaService.getSchema(schemaName, schemaVersion).thenAccept(schemaAndMetadata -> {
-            if (schemaAndMetadata == null) {
-                commandSender.sendGetSchemaErrorResponse(requestId, ServerError.TopicNotFound,
-                        String.format("Topic not found or no-schema %s", topic));
-            } else {
-                commandSender.sendGetSchemaResponse(requestId,
-                        SchemaInfoUtil.newSchemaInfo(schemaName, schemaAndMetadata.schema), schemaAndMetadata.version);
-            }
-        }).exceptionally(ex -> {
-            commandSender.sendGetSchemaErrorResponse(requestId, ServerError.UnknownError, ex.getMessage());
-            return null;
-        });
+        // Producers, consumers and readers fetch the schema of a topic they have looked up, so LOOKUP is a
+        // permission every legitimate caller already holds.
+        isTopicOperationAllowed(topicName, TopicOperation.LOOKUP, authenticationData, originalAuthData)
+                .thenCompose(isAuthorized -> {
+                    if (!isAuthorized) {
+                        commandSender.sendGetSchemaErrorResponse(requestId, ServerError.AuthorizationError,
+                                "Client is not authorized to get the schema of " + topic);
+                        return CompletableFuture.completedFuture(null);
+                    }
+                    return schemaService.getSchema(schemaName, requestedVersion).thenAccept(schemaAndMetadata -> {
+                        if (schemaAndMetadata == null) {
+                            commandSender.sendGetSchemaErrorResponse(requestId, ServerError.TopicNotFound,
+                                    String.format("Topic not found or no-schema %s", topic));
+                        } else {
+                            commandSender.sendGetSchemaResponse(requestId,
+                                    SchemaInfoUtil.newSchemaInfo(schemaName, schemaAndMetadata.schema),
+                                    schemaAndMetadata.version);
+                        }
+                    });
+                }).exceptionally(ex -> {
+                    commandSender.sendGetSchemaErrorResponse(requestId, ServerError.UnknownError, ex.getMessage());
+                    return null;
+                });
     }
 
     @Override
@@ -2809,9 +2823,28 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
         }
         long requestId = commandGetOrCreateSchema.getRequestId();
         final String topicName = commandGetOrCreateSchema.getTopic();
+        final TopicName parsedTopicName;
+        try {
+            parsedTopicName = TopicName.get(topicName);
+        } catch (Throwable t) {
+            commandSender.sendGetOrCreateSchemaErrorResponse(requestId, ServerError.InvalidTopicName,
+                    t.getMessage());
+            return;
+        }
         SchemaData schemaData = getSchema(commandGetOrCreateSchema.getSchema());
         SchemaData schema = schemaData.getType() == SchemaType.NONE ? null : schemaData;
-        service.getTopicIfExists(topicName).thenAccept(topicOpt -> {
+        // Adding a schema version changes what the topic's producers may send, so it takes PRODUCE, as the
+        // REST schema upload does.
+        CompletableFuture<Optional<Topic>> topicFuture =
+                isTopicOperationAllowed(parsedTopicName, TopicOperation.PRODUCE, authenticationData, originalAuthData)
+                        .thenCompose(isAuthorized -> {
+                            if (!isAuthorized) {
+                                return CompletableFuture.failedFuture(new BrokerServiceException.NotAuthorizedException(
+                                        "Client is not authorized to add a schema to " + topicName));
+                            }
+                            return service.getTopicIfExists(topicName);
+                        });
+        topicFuture.thenAccept(topicOpt -> {
             if (topicOpt.isPresent()) {
                 Topic topic = topicOpt.get();
                 CompletableFuture<SchemaVersion> schemaVersionFuture = tryAddSchema(topic, schema);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
index 968fd442005e..997bd19cf077 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java
@@ -72,6 +72,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -116,9 +117,11 @@ import org.apache.pulsar.broker.namespace.TopicExistsInfo;
 import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException;
 import org.apache.pulsar.broker.service.ServerCnx.State;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
 import org.apache.pulsar.broker.service.utils.ClientChannelHelper;
 import org.apache.pulsar.broker.testcontext.PulsarTestContext;
 import org.apache.pulsar.client.api.ProducerAccessMode;
+import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.transaction.TxnID;
 import org.apache.pulsar.common.api.AuthData;
 import org.apache.pulsar.common.api.proto.AuthMethod;
@@ -134,6 +137,8 @@ import org.apache.pulsar.common.api.proto.CommandEndTxnOnPartitionResponse;
 import org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscriptionResponse;
 import org.apache.pulsar.common.api.proto.CommandEndTxnResponse;
 import org.apache.pulsar.common.api.proto.CommandError;
+import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchemaResponse;
+import org.apache.pulsar.common.api.proto.CommandGetSchemaResponse;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse;
 import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse;
@@ -3825,6 +3830,96 @@ public class ServerCnxTest {
         return authorizationService;
     }
 
+    private AtomicInteger countSchemaReads() {
+        AtomicInteger schemaReads = new AtomicInteger();
+        SchemaRegistryService schemaRegistryService = pulsar.getSchemaRegistryService();
+        doAnswer(invocation -> {
+            schemaReads.incrementAndGet();
+            return CompletableFuture.completedFuture(null);
+        }).when(schemaRegistryService).getSchema(any(), any());
+        return schemaReads;
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetSchemaRequiresLookupPermission() throws Exception {
+        AuthorizationService authorizationService = mockTopicAuthorization(false);
+        AtomicInteger schemaReads = countSchemaReads();
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newGetSchema(1L, successTopicName, Optional.empty()));
+        CommandGetSchemaResponse response = (CommandGetSchemaResponse) getResponse();
+
+        assertEquals(response.getErrorCode(), ServerError.AuthorizationError);
+        verify(authorizationService, times(1)).allowTopicOperationAsync(eq(TopicName.get(successTopicName)),
+                eq(TopicOperation.LOOKUP), any(), any(), any(), any());
+        assertEquals(schemaReads.get(), 0);
+
+        channel.finish();
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetSchemaWithLookupPermission() throws Exception {
+        mockTopicAuthorization(true);
+        AtomicInteger schemaReads = countSchemaReads();
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newGetSchema(1L, successTopicName, Optional.empty()));
+        CommandGetSchemaResponse response = (CommandGetSchemaResponse) getResponse();
+
+        // The registry holds no schema, so an authorized request reaches it and gets TopicNotFound.
+        assertEquals(response.getErrorCode(), ServerError.TopicNotFound);
+        assertEquals(schemaReads.get(), 1);
+
+        channel.finish();
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetOrCreateSchemaRequiresProducePermission() throws Exception {
+        AuthorizationService authorizationService = mockTopicAuthorization(false);
+        AtomicBoolean topicLookedUp = new AtomicBoolean();
+        doAnswer(invocation -> {
+            topicLookedUp.set(true);
+            return CompletableFuture.completedFuture(Optional.empty());
+        }).when(brokerService).getTopicIfExists(any(String.class));
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newGetOrCreateSchema(1L, successTopicName,
+                Schema.STRING.getSchemaInfo()));
+        CommandGetOrCreateSchemaResponse response = (CommandGetOrCreateSchemaResponse) getResponse();
+
+        assertEquals(response.getErrorCode(), ServerError.AuthorizationError);
+        verify(authorizationService, times(1)).allowTopicOperationAsync(eq(TopicName.get(successTopicName)),
+                eq(TopicOperation.PRODUCE), any(), any(), any(), any());
+        assertFalse(topicLookedUp.get());
+
+        channel.finish();
+    }
+
+    @Test(timeOut = 30000)
+    public void testGetOrCreateSchemaWithProducePermission() throws Exception {
+        mockTopicAuthorization(true);
+        AtomicBoolean topicLookedUp = new AtomicBoolean();
+        doAnswer(invocation -> {
+            topicLookedUp.set(true);
+            return CompletableFuture.completedFuture(Optional.empty());
+        }).when(brokerService).getTopicIfExists(any(String.class));
+        resetChannel();
+        setChannelConnected();
+
+        channel.writeInbound(Commands.newGetOrCreateSchema(1L, successTopicName,
+                Schema.STRING.getSchemaInfo()));
+        CommandGetOrCreateSchemaResponse response = (CommandGetOrCreateSchemaResponse) getResponse();
+
+        // No topic is loaded, so an authorized request reaches the topic lookup and gets TopicNotFound.
+        assertEquals(response.getErrorCode(), ServerError.TopicNotFound);
+        assertTrue(topicLookedUp.get());
+
+        channel.finish();
+    }
+
     @Test(timeOut = 30000)
     public void testAddPartitionToTxnRefusedWhenOnePartitionIsDenied() throws Exception {
         AuthorizationService authorizationService = mockTopicAuthorization(true);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java
index d6210767ed1f..1d3eefff3b86 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/utils/ClientChannelHelper.java
@@ -37,6 +37,8 @@ import org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscriptionResponse;
 import org.apache.pulsar.common.api.proto.CommandEndTxnResponse;
 import org.apache.pulsar.common.api.proto.CommandError;
 import org.apache.pulsar.common.api.proto.CommandFlow;
+import org.apache.pulsar.common.api.proto.CommandGetOrCreateSchemaResponse;
+import org.apache.pulsar.common.api.proto.CommandGetSchemaResponse;
 import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse;
 import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse;
 import org.apache.pulsar.common.api.proto.CommandMessage;
@@ -175,6 +177,16 @@ public class ClientChannelHelper {
             queue.offer(new CommandWatchTopicListSuccess().copyFrom(commandWatchTopicListSuccess));
         }
 
+        @Override
+        protected void handleGetSchemaResponse(CommandGetSchemaResponse response) {
+            queue.offer(new CommandGetSchemaResponse().copyFrom(response));
+        }
+
+        @Override
+        protected void handleGetOrCreateSchemaResponse(CommandGetOrCreateSchemaResponse response) {
+            queue.offer(new CommandGetOrCreateSchemaResponse().copyFrom(response));
+        }
+
         @Override
         protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse response) {
             queue.offer(new CommandPartitionedTopicMetadataResponse().copyFrom(response));


(pulsar) 15/27: [fix][client] Stop tracking ack timeouts on non-persistent topics (#26548)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 371af5c03e8bd4b2abb8d8fda71e554381858339
Author: SongOf <46...@users.noreply.github.com>
AuthorDate: Sat Sep 12 06:02:57 2026 +0800

    [fix][client] Stop tracking ack timeouts on non-persistent topics (#26548)
    
    Co-authored-by: maxlisongsong <ma...@didiglobal.com>
    (cherry picked from commit a9eedcdc933b165f8a850d939749c8957528d2b4)
---
 .../apache/pulsar/client/impl/ConsumerBase.java    | 15 ++++-
 .../apache/pulsar/client/impl/ConsumerImpl.java    | 23 ++++++-
 .../pulsar/client/impl/ZeroQueueConsumerImpl.java  |  6 +-
 .../pulsar/client/impl/ConsumerImplTest.java       | 72 +++++++++++++++++++++-
 4 files changed, 109 insertions(+), 7 deletions(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
index 7f18c32ed62d..8b0af908b3cb 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java
@@ -243,9 +243,18 @@ public abstract class ConsumerBase<T> extends HandlerState implements Consumer<T
         }
     }
 
+    /**
+     * Whether unacknowledged messages should be tracked so that the ack timeout can redeliver them.
+     * Overridden by {@link ConsumerImpl} to exclude non-persistent topics, where the broker keeps nothing to
+     * replay and a timeout can therefore never produce a redelivery.
+     */
+    protected boolean isAckTimeoutTrackingEnabled() {
+        return conf.getAckTimeoutMillis() > 0;
+    }
+
     // if listener is not null, we will track unAcked msg in callMessageListener
     protected void trackUnAckedMsgIfNoListener(MessageId messageId, int redeliveryCount) {
-        if (listener == null) {
+        if (listener == null && isAckTimeoutTrackingEnabled()) {
             unAckedMessageTracker.add(messageId, redeliveryCount);
         }
     }
@@ -1291,7 +1300,9 @@ public abstract class ConsumerBase<T> extends HandlerState implements Consumer<T
             } else {
                 id = msg.getMessageId();
             }
-            unAckedMessageTracker.add(id, msg.getRedeliveryCount());
+            if (isAckTimeoutTrackingEnabled()) {
+                unAckedMessageTracker.add(id, msg.getRedeliveryCount());
+            }
             beforeConsume(msg);
             Optional<EncryptionContext> encryptionCtx = msg.getEncryptionCtx();
             if (decryptFailListener != null && encryptionCtx.isPresent() && encryptionCtx.get().isEncrypted()) {
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index e1db640c2396..7074ffa9a595 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -386,6 +386,11 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
         } else {
             this.acknowledgmentsGroupingTracker =
                     NonPersistentAcknowledgmentGroupingTracker.of();
+            if (conf.getAckTimeoutMillis() > 0) {
+                log.warn("[{}] Ignoring the configured ack timeout of {} ms: a non-persistent topic keeps nothing to"
+                        + " replay, so unacknowledged messages can never be redelivered", topic,
+                        conf.getAckTimeoutMillis());
+            }
         }
 
         if (conf.getDeadLetterPolicy() != null) {
@@ -1916,8 +1921,20 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
             trackMessage(messageId, 0);
     }
 
+    /**
+     * Never track on a non-persistent topic. The broker stores nothing to replay there, so an ack timeout can
+     * never produce a redelivery, and an ack does not clear the tracker either: the consumer installs
+     * {@link NonPersistentAcknowledgmentGroupingTracker}, whose {@code addAcknowledgment} is a no-op, while the
+     * tracker is only cleared from the persistent one. Tracking would therefore fill up even for an application
+     * that acks everything, and the resulting timeout clears the receive queue, destroying messages for good.
+     */
+    @Override
+    protected boolean isAckTimeoutTrackingEnabled() {
+        return super.isAckTimeoutTrackingEnabled() && topicName.isPersistent();
+    }
+
     protected void trackMessage(MessageId messageId, int redeliveryCount) {
-        if (conf.getAckTimeoutMillis() > 0 && messageId instanceof MessageIdImpl) {
+        if (isAckTimeoutTrackingEnabled() && messageId instanceof MessageIdImpl) {
             MessageId id = MessageIdAdvUtils.discardBatch(messageId);
             if (hasParentConsumer) {
                 //TODO: check parent consumer here
@@ -2102,7 +2119,9 @@ public class ConsumerImpl<T> extends ConsumerBase<T> implements ConnectionHandle
                         partitionIndex);
             }
             MessageId m = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex);
-            unAckedMessageTracker.add(m, redeliveryCount);
+            if (isAckTimeoutTrackingEnabled()) {
+                unAckedMessageTracker.add(m, redeliveryCount);
+            }
             return DecryptResult.discard();
         default:
             log.warn("[{}][{}][{}] Invalid crypto failure state found, continue message consumption.", topic,
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
index 26fb1e0c8caa..bc8baa49da06 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java
@@ -174,8 +174,10 @@ public class ZeroQueueConsumerImpl<T> extends ConsumerImpl<T> {
                 }
                 waitingOnListenerForZeroQueueSize = true;
                 trackMessage(message);
-                unAckedMessageTracker.add(
-                        MessageIdAdvUtils.discardBatch(message.getMessageId()), message.getRedeliveryCount());
+                if (isAckTimeoutTrackingEnabled()) {
+                    unAckedMessageTracker.add(
+                            MessageIdAdvUtils.discardBatch(message.getMessageId()), message.getRedeliveryCount());
+                }
                 if (decryptFailListener != null
                         && message.getEncryptionCtx().isPresent()
                         && message.getEncryptionCtx().get().isEncrypted()
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
index a427e35a4c45..03275a2736c6 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java
@@ -48,6 +48,7 @@ import lombok.Cleanup;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageIdAdv;
+import org.apache.pulsar.client.api.MessageListener;
 import org.apache.pulsar.client.api.MessagePayload;
 import org.apache.pulsar.client.api.Messages;
 import org.apache.pulsar.client.api.PulsarClient;
@@ -81,6 +82,10 @@ public class ConsumerImplTest {
     }
 
     private void createConsumer(ConsumerConfigurationData consumerConf) {
+        createConsumer(consumerConf, topic);
+    }
+
+    private void createConsumer(ConsumerConfigurationData consumerConf, String topicName) {
         executorProvider = new ExecutorProvider(1, "ConsumerImplTest");
         internalExecutor = Executors.newSingleThreadScheduledExecutor();
 
@@ -91,7 +96,7 @@ public class ConsumerImplTest {
         CompletableFuture<Consumer<byte[]>> subscribeFuture = new CompletableFuture<>();
 
         consumerConf.setSubscriptionName("test-sub");
-        consumer = ConsumerImpl.newConsumerImpl(client, topic, consumerConf,
+        consumer = ConsumerImpl.newConsumerImpl(client, topicName, consumerConf,
                 executorProvider, -1, false, subscribeFuture, null, null, null,
                 true);
         consumer.setState(HandlerState.State.Ready);
@@ -393,4 +398,69 @@ public class ConsumerImplTest {
             context.recycle();
         }
     }
+
+    /**
+     * On a non-persistent topic the broker keeps nothing to replay, so an ack timeout can never lead to a
+     * redelivery: {@code NonPersistentSubscription.redeliverUnacknowledgedMessages} is a no-op. Tracking
+     * messages anyway is worse than useless, because acks do not clear the tracker either — the consumer
+     * installs {@code NonPersistentAcknowledgmentGroupingTracker}, whose {@code addAcknowledgment} is a no-op,
+     * and the tracker is only cleared from the persistent one. The tracker therefore fills up even for an
+     * application that acks every message, and once it times out the consumer clears its receive queue,
+     * destroying messages that nothing can replay.
+     */
+    @Test
+    public void testMessagesAreNotTrackedForAckTimeoutOnNonPersistentTopic() {
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setAckTimeoutMillis(TimeUnit.SECONDS.toMillis(10));
+        createConsumer(conf, "non-persistent://tenant/ns1/ack-timeout-topic");
+
+        consumer.trackMessage(new MessageIdImpl(1L, 1L, -1), 0);
+
+        Assert.assertTrue(consumer.getUnAckedMessageTracker().isEmpty(),
+                "a message was tracked for ack timeout on a non-persistent topic, where an ack never clears"
+                        + " the tracker and a redelivery can never happen");
+    }
+
+    /** The same configuration must keep working on a persistent topic, where redelivery is possible. */
+    @Test
+    public void testMessagesAreStillTrackedForAckTimeoutOnPersistentTopic() {
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setAckTimeoutMillis(TimeUnit.SECONDS.toMillis(10));
+        createConsumer(conf, "persistent://tenant/ns1/ack-timeout-topic");
+
+        consumer.trackMessage(new MessageIdImpl(1L, 1L, -1), 0);
+
+        Assert.assertFalse(consumer.getUnAckedMessageTracker().isEmpty(),
+                "messages are no longer tracked for ack timeout on a persistent topic");
+    }
+
+    /**
+     * The listener dispatch path adds to the tracker itself, bypassing {@code trackMessage} entirely
+     * ({@code trackUnAckedMsgIfNoListener} only adds when no listener is set). Consumers with a
+     * {@code messageListener} are the common case, so this path must honour the same rule.
+     */
+    @Test
+    public void testMessagesAreNotTrackedForAckTimeoutOnNonPersistentTopicWithListener() {
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setAckTimeoutMillis(TimeUnit.SECONDS.toMillis(10));
+        conf.setMessageListener((MessageListener<byte[]>) (c, msg) -> { });
+        createConsumer(conf, "non-persistent://tenant/ns1/ack-timeout-listener-topic");
+
+        Assert.assertFalse(consumer.isAckTimeoutTrackingEnabled(),
+                "ack timeout tracking is still enabled on a non-persistent topic, so the listener dispatch path"
+                        + " would keep filling the tracker");
+    }
+
+    /** On a persistent topic the listener path must keep tracking. */
+    @Test
+    public void testMessagesAreStillTrackedForAckTimeoutOnPersistentTopicWithListener() {
+        ConsumerConfigurationData<byte[]> conf = new ConsumerConfigurationData<>();
+        conf.setAckTimeoutMillis(TimeUnit.SECONDS.toMillis(10));
+        conf.setMessageListener((MessageListener<byte[]>) (c, msg) -> { });
+        createConsumer(conf, "persistent://tenant/ns1/ack-timeout-listener-topic");
+
+        Assert.assertTrue(consumer.isAckTimeoutTrackingEnabled(),
+                "ack timeout tracking was disabled on a persistent topic");
+    }
+
 }


(pulsar) 05/27: [fix][build] Upgrade Conscrypt to 2.6.3 (#26660)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 5e87b6ae9414caafeaec4e8c8e60a2e068a17eb2
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Sun Sep 20 04:43:57 2026 +0300

    [fix][build] Upgrade Conscrypt to 2.6.3 (#26660)
    
    (cherry picked from commit 911b475e2740a51f7eb9a2e08d95f517d97fcdbf)
---
 distribution/server/src/assemble/LICENSE.bin.txt | 2 +-
 distribution/shell/src/assemble/LICENSE.bin.txt  | 2 +-
 pom.xml                                          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index 22e4e6ce6038..61cf19eaa5d8 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -261,7 +261,7 @@ The Apache Software License, Version 2.0
      - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.10.jar
      - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.10.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-3.2.4.jar
- * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.2.jar
+ * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.3.jar
  * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar
  * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.59.2.jar
  * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.6.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index df5983f0aef9..eb7ec5c85065 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -325,7 +325,7 @@ The Apache Software License, Version 2.0
      - jackson-datatype-jsr310-2.18.10.jar
      - jackson-module-parameter-names-2.18.10.jar
  * Caffeine -- caffeine-3.2.4.jar
- * Conscrypt -- conscrypt-openjdk-uber-2.6.2.jar
+ * Conscrypt -- conscrypt-openjdk-uber-2.6.3.jar
  * Gson
     - gson-2.13.2.jar
  * Guava
diff --git a/pom.xml b/pom.xml
index a677d3869f5d..d806623815f1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,7 +194,7 @@ flexible messaging model and an intuitive client API.</description>
     <jetty.version>12.1.12</jetty.version>
     <!-- Jetty 9 is used in tiered-storage/file-system tests and pulsar-io/alluxio -->
     <jetty9.version>9.4.58.v20250814</jetty9.version>
-    <conscrypt.version>2.6.2</conscrypt.version>
+    <conscrypt.version>2.6.3</conscrypt.version>
     <jersey.version>2.42</jersey.version>
     <athenz.version>1.10.62</athenz.version>
     <prometheus.version>0.16.0</prometheus.version>


(pulsar) 06/27: [fix][cli] Correct admin command documentation generation (#26649)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 8319acf0d4ed38ca8f0d86c72d071dc3e68d6288
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Sun Sep 20 05:15:27 2026 +0300

    [fix][cli] Correct admin command documentation generation (#26649)
    
    (cherry picked from commit e11185bd681ef38d59f15d16a7d435dcad9d075c)
---
 .../pulsar/admin/cli/CmdGenerateDocument.java      | 26 +++----
 .../pulsar/admin/cli/TestCmdGenerateDocument.java  | 89 ++++++++++++++++++++++
 2 files changed, 102 insertions(+), 13 deletions(-)

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
index 3f728ca73ea6..67bae05c2f4b 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdGenerateDocument.java
@@ -56,11 +56,12 @@ public class CmdGenerateDocument extends CmdBase {
         @Parameters(description = "Please specify the module name, if not, documents will be generated for all modules."
                 + "Optional modules(clusters, tenants, brokers, broker-stats, namespaces, topics, schemas, bookies,"
                 + "functions, ns-isolation-policy, resource-quotas, functions, sources, sinks)")
-        private java.util.List<String> modules;
+        private List<String> modules;
 
         @Override
         void run() throws PulsarAdminException {
             StringBuilder sb = new StringBuilder();
+            generatedModule.clear();
             if (modules == null || modules.isEmpty()) {
                 pulsarAdminCommandSpec.parent().subcommands().forEach((k, v) ->
                         this.generateDocument(sb, k, v)
@@ -74,6 +75,7 @@ public class CmdGenerateDocument extends CmdBase {
                     this.generateDocument(sb, module, commandLine);
                 });
             }
+            print(sb.toString());
         }
 
         private boolean needsLangSupport(String module, String subK) {
@@ -85,11 +87,9 @@ public class CmdGenerateDocument extends CmdBase {
 
         private void generateDocument(StringBuilder sb, String module, CommandLine obj) {
             // Filter the deprecated command
-            if (generatedModule.contains(module)) {
+            if (!generatedModule.add(obj.getCommandName())) {
                 return;
             }
-            String commandName = obj.getCommandName();
-            generatedModule.add(commandName);
 
             sb.append("# ").append(module).append("\n\n");
             sb.append(getCommandDescription(obj)).append("\n");
@@ -103,7 +103,9 @@ public class CmdGenerateDocument extends CmdBase {
                 sb.append("**Command:**\n\n");
                 sb.append("```shell\n$ pulsar-admin ").append(module).append(" ")
                         .append(subK).append(" options").append("\n```\n\n");
-                List<ArgSpec> options = obj.getCommandSpec().args();
+                List<ArgSpec> options = subV.getCommandSpec().args().stream()
+                        .filter(arg -> arg instanceof OptionSpec && !arg.hidden())
+                        .toList();
                 if (options.size() > 0) {
                     sb.append("**Options:**\n\n");
                     sb.append("|Flag|Description|Default|");
@@ -115,21 +117,19 @@ public class CmdGenerateDocument extends CmdBase {
                     }
                 }
                 options.forEach(ele -> {
-                    if (ele.hidden() || !(ele instanceof OptionSpec)) {
-                        return;
-                    }
-
                     String argDescription = getArgDescription(ele);
                     String[] descriptions = argDescription.replace("\n", " ").split(" #");
                     sb.append("| `").append(Arrays.toString(((OptionSpec) ele).names()))
                             .append("` | ").append(descriptions[0])
-                            .append("|").append(ele.defaultValue()).append("|");
-                    if (needsLangSupport(module, subK) && descriptions.length > 1) {
-                        sb.append(descriptions[1]);
+                            .append("|").append(ele.defaultValueString());
+                    if (needsLangSupport(module, subK)) {
+                        sb.append("|");
+                        if (descriptions.length > 1) {
+                            sb.append(descriptions[1]);
+                        }
                     }
                     sb.append("|\n");
                 });
-                System.out.println(sb);
             });
         }
     }
diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdGenerateDocument.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdGenerateDocument.java
new file mode 100644
index 000000000000..3a71872ed4f6
--- /dev/null
+++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdGenerateDocument.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.admin.cli;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.function.Consumer;
+import org.testng.annotations.Test;
+import picocli.CommandLine;
+import picocli.CommandLine.Model.CommandSpec;
+
+public class TestCmdGenerateDocument {
+    private String generate(Consumer<CommandLine> action) {
+        PrintStream original = System.out;
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+        try (PrintStream output = new PrintStream(bytes, true, StandardCharsets.UTF_8)) {
+            System.setOut(output);
+            CommandLine root = new CommandLine(CommandSpec.create().name("pulsar-admin"));
+            root.addSubcommand("brokers", new CmdBrokers(() -> null).getCommander(), "broker-alias");
+            root.addSubcommand("ns-isolation-policy", new CmdNamespaceIsolationPolicy(() -> null).getCommander());
+            root.addSubcommand("documents", new CmdGenerateDocument(() -> null).getCommander());
+            action.accept(root);
+            return bytes.toString(StandardCharsets.UTF_8);
+        } finally {
+            System.setOut(original);
+        }
+    }
+
+    @Test
+    public void testEachSubcommandIsPrintedOnce() {
+        String document = generate(root -> assertThat(root.execute("documents", "generate", "brokers")).isZero());
+        assertThat(document.lines().filter(line -> line.equals("# brokers")).count()).isEqualTo(1);
+        assertThat(document).containsOnlyOnce("## list\n");
+        assertThat(document).containsOnlyOnce("## shutdown\n");
+    }
+
+    @Test
+    public void testSubcommandOptionsAndTableColumns() {
+        String document = generate(root ->
+                assertThat(root.execute("documents", "generate", "ns-isolation-policy")).isZero());
+        assertThat(document).contains("--primary", "--secondary", "--unload-scope");
+        assertThat(document.lines().filter(line -> line.startsWith("|")))
+                .allSatisfy(line -> assertThat(line.chars().filter(c -> c == '|').count()).isEqualTo(4));
+    }
+
+    @Test
+    public void testMultipleModulesAndAliases() {
+        String document = generate(root -> assertThat(root.execute("documents", "generate",
+                "brokers", "broker-alias", "ns-isolation-policy")).isZero());
+        assertThat(document.lines().filter(line -> line.equals("# brokers")).count()).isEqualTo(1);
+        assertThat(document.lines().filter(line -> line.equals("# ns-isolation-policy")).count()).isEqualTo(1);
+        assertThat(document).doesNotContain("# broker-alias\n");
+    }
+
+    @Test
+    public void testAllModules() {
+        String document = generate(root -> assertThat(root.execute("documents", "generate")).isZero());
+        assertThat(document.lines().filter(line -> line.equals("# brokers")).count()).isEqualTo(1);
+        assertThat(document.lines().filter(line -> line.equals("# ns-isolation-policy")).count()).isEqualTo(1);
+        assertThat(document).doesNotContain("# broker-alias\n");
+    }
+
+    @Test
+    public void testRepeatedInvocation() {
+        String document = generate(root -> {
+            assertThat(root.execute("documents", "generate", "brokers")).isZero();
+            assertThat(root.execute("documents", "generate", "brokers")).isZero();
+        });
+        assertThat(document.lines().filter(line -> line.equals("# brokers")).count()).isEqualTo(2);
+    }
+}


(pulsar) 12/27: [fix][test] Join assignment tailer before asserting its last message ID (#26536)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 5d3cd8b5e7ec700eda2f03a9674dd0401f21f803
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Fri Sep 11 01:14:28 2026 +0300

    [fix][test] Join assignment tailer before asserting its last message ID (#26536)
    
    (cherry picked from commit 3fb6464da3b818f1d0fd4eff124d190345e8650c)
---
 .../worker/FunctionAssignmentTailerTest.java       | 35 +++++++---------------
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
index 14bd0cde268f..c1f9e2b73535 100644
--- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionAssignmentTailerTest.java
@@ -54,6 +54,7 @@ import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory;
 import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig;
 import org.apache.pulsar.functions.utils.FunctionCommon;
+import org.awaitility.Awaitility;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 import org.testng.Assert;
@@ -286,36 +287,20 @@ public class FunctionAssignmentTailerTest {
         FunctionAssignmentTailer functionAssignmentTailer =
                 spy(new FunctionAssignmentTailer(functionRuntimeManager, readerBuilder, workerConfig, errorNotifier));
 
-        functionAssignmentTailer.start();
+        try (functionAssignmentTailer) {
+            functionAssignmentTailer.start();
 
-        messageList.add(message1);
-        for (int i = 0; i < 10; i++) {
-            try {
-                verify(functionRuntimeManager, times(1)).processAssignmentMessage(eq(message1));
-                break;
-            } catch (org.mockito.exceptions.verification.WantedButNotInvoked e) {
-                if (i == 9) {
-                    throw e;
-                }
-            }
-            Thread.sleep(200);
-        }
+            messageList.add(message1);
+            Awaitility.await().untilAsserted(() ->
+                    verify(functionRuntimeManager, times(1)).processAssignmentMessage(eq(message1)));
 
-        messageList.add(message2);
-        for (int i = 0; i < 10; i++) {
-            try {
-                verify(functionRuntimeManager, times(1)).processAssignmentMessage(eq(message2));
-                break;
-            } catch (org.mockito.exceptions.verification.WantedButNotInvoked e) {
-                if (i == 9) {
-                    throw e;
-                }
-            }
-            Thread.sleep(200);
+            messageList.add(message2);
+            Awaitility.await().untilAsserted(() ->
+                    verify(functionRuntimeManager, times(1)).processAssignmentMessage(eq(message2)));
         }
 
+        // Verification observes method entry; close joins the tailer after it updates the last message id.
         Assert.assertEquals(functionAssignmentTailer.getLastMessageId(), message2.getMessageId());
-        functionAssignmentTailer.close();
     }
 
     @Test(timeOut = 10000)


(pulsar) 11/27: [fix][ml] Fix batch ACK index loss when recovering cursor from MetadataStore (#26474)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 5daa008bf63afadef7411a4a360357af64517182
Author: void-ptr974 <vo...@protonmail.com>
AuthorDate: Wed Sep 9 23:12:11 2026 +0800

    [fix][ml] Fix batch ACK index loss when recovering cursor from MetadataStore (#26474)
    
    (cherry picked from commit 5a47023fa3b29a432649352c0d056a4d3f847203)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |   5 +
 .../impl/ManagedCursorBatchAckRecoveryTest.java    | 203 +++++++++++++++++++++
 ...gedCursorBatchAckMetadataStoreRecoveryTest.java | 195 ++++++++++++++++++++
 3 files changed, 403 insertions(+)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 85357e5067d0..024b870bb6fc 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -578,6 +578,11 @@ public class ManagedCursorImpl implements ManagedCursor {
                         recoverIndividualDeletedMessages(info.getIndividualDeletedMessagesCount(),
                                 info::getIndividualDeletedMessageAt);
                     }
+                    if (getConfig().isDeletionAtBatchIndexLevelEnabled()
+                            && info.getBatchedEntryDeletionIndexInfosCount() > 0) {
+                        recoverBatchDeletedIndexes(info.getBatchedEntryDeletionIndexInfosCount(),
+                                info::getBatchedEntryDeletionIndexInfoAt);
+                    }
 
                     Map<String, Long> recoveredProperties =
                             recoverProperties(info.getPropertiesCount(), info::getPropertyAt);
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
new file mode 100644
index 000000000000..210c6a02a7a8
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckRecoveryTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
+import org.apache.bookkeeper.mledger.proto.BatchedEntryDeletionIndexInfo;
+import org.apache.bookkeeper.mledger.proto.ManagedCursorInfo;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.apache.pulsar.metadata.api.Stat;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+/** Verifies recovery of partial batch acknowledgments from cursor metadata. */
+@Test(timeOut = 30000)
+public class ManagedCursorBatchAckRecoveryTest extends MockedBookKeeperTestCase {
+
+    @DataProvider(name = "batchRecovery")
+    public Object[][] batchRecovery() {
+        return new Object[][] {{1, false}, {2, false}, {3, false}, {1, true}, {2, true}, {3, true}};
+    }
+
+    @DataProvider(name = "booleans")
+    public Object[][] booleans() {
+        return new Object[][] {{false}, {true}};
+    }
+
+    @Test(dataProvider = "batchRecovery")
+    public void testPartialBatchAckPersistenceRetainsConfiguredCount(int batchCount, boolean metadataStore)
+            throws Exception {
+        ManagedLedgerConfig config = new ManagedLedgerConfig()
+                .setMaxUnackedRangesToPersistInMetadataStore(metadataStore ? 20 : -1);
+        config.setDeletionAtBatchIndexLevelEnabled(true);
+        config.setMaxBatchDeletedIndexToPersist(2);
+        String name = "tenant/ns/persistent/partial-batch-recovery-" + batchCount + "-" + metadataStore;
+        @Cleanup
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(name, config);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub");
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i <= batchCount; i++) {
+            positions.add(ledger.addEntry(new byte[] {(byte) i}));
+        }
+        long[][] ackSets = {{2L}, {Long.MIN_VALUE, 1L}, {5L, 0L, 3L}};
+        for (int i = 1; i < positions.size(); i++) {
+            Position position = positions.get(i);
+            cursor.delete(AckSetStateUtil.createPositionWithAckSet(
+                    position.getLedgerId(), position.getEntryId(), ackSets[i - 1]));
+        }
+
+        // Advance mark-delete, then close to persist and reload the actual durable state.
+        cursor.delete(positions.get(0));
+        ledger.close();
+        ManagedCursorInfo persisted = storedCursorInfo(ledger);
+        if (metadataStore) {
+            assertThat(persisted.getCursorsLedgerId()).isEqualTo(-1L);
+            int expectedPersistedCount = Math.min(batchCount, 2);
+            assertThat(persisted.getBatchedEntryDeletionIndexInfosCount())
+                    .as("the partial ACK records were actually written before recovery")
+                    .isEqualTo(expectedPersistedCount);
+            for (int i = 0; i < expectedPersistedCount; i++) {
+                BatchedEntryDeletionIndexInfo indexInfo = persisted.getBatchedEntryDeletionIndexInfoAt(i);
+                assertThat(indexInfo.getPosition().getLedgerId()).isEqualTo(positions.get(i + 1).getLedgerId());
+                assertThat(indexInfo.getPosition().getEntryId()).isEqualTo(positions.get(i + 1).getEntryId());
+                assertThat(deleteSet(indexInfo)).containsExactly(ackSets[i]);
+            }
+        } else {
+            assertThat(persisted.getCursorsLedgerId()).isGreaterThanOrEqualTo(0);
+        }
+
+        @Cleanup
+        ManagedLedgerImpl reopened = (ManagedLedgerImpl) factory.open(name, config);
+        ManagedCursorImpl recovered = (ManagedCursorImpl) reopened.openCursor("sub");
+        assertThat(recovered).isNotSameAs(cursor);
+        for (int i = 1; i <= batchCount; i++) {
+            long[] recoveredAckSet = recovered.getBatchPositionAckSet(positions.get(i));
+            if (i <= 2) {
+                assertThat(recoveredAckSet).containsExactly(ackSets[i - 1]);
+            } else {
+                assertThat(recoveredAckSet).as("ACK state above the configured batch cap is truncated").isNull();
+            }
+        }
+    }
+
+    @Test
+    public void testMetadataStoreRecoveryPreservesAllCursorState() throws Exception {
+        ManagedLedgerConfig config = new ManagedLedgerConfig()
+                .setMaxUnackedRangesToPersistInMetadataStore(20);
+        config.setDeletionAtBatchIndexLevelEnabled(true);
+        String name = "tenant/ns/persistent/combined-metadata-store-recovery";
+        @Cleanup
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(name, config);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub");
+        List<Position> positions = new ArrayList<>();
+        for (int i = 0; i < 5; i++) {
+            positions.add(ledger.addEntry(new byte[] {(byte) i}));
+        }
+
+        cursor.markDelete(positions.get(0), Map.of("mark-delete-property", 7L));
+        cursor.setCursorProperties(Map.of("cursor-property", "value")).get(5, TimeUnit.SECONDS);
+        cursor.delete(positions.get(2));
+        long[] ackSet = {Long.MIN_VALUE, 1L};
+        cursor.delete(AckSetStateUtil.createPositionWithAckSet(
+                positions.get(4).getLedgerId(), positions.get(4).getEntryId(), ackSet));
+        ledger.close();
+
+        ManagedCursorInfo persisted = storedCursorInfo(ledger);
+        assertThat(persisted.getCursorsLedgerId()).isEqualTo(-1L);
+        assertThat(persisted.getIndividualDeletedMessagesCount()).isEqualTo(1);
+        assertThat(persisted.getBatchedEntryDeletionIndexInfosCount()).isEqualTo(1);
+        assertThat(deleteSet(persisted.getBatchedEntryDeletionIndexInfoAt(0))).containsExactly(ackSet);
+
+        @Cleanup
+        ManagedLedgerImpl reopened = (ManagedLedgerImpl) factory.open(name, config);
+        ManagedCursorImpl recovered = (ManagedCursorImpl) reopened.openCursor("sub");
+        assertThat(recovered).isNotSameAs(cursor);
+        assertThat(recovered.getMarkDeletedPosition()).isEqualTo(positions.get(0));
+        assertThat(recovered.getProperties()).containsExactlyEntriesOf(Map.of("mark-delete-property", 7L));
+        assertThat(recovered.getCursorProperties()).containsExactlyEntriesOf(Map.of("cursor-property", "value"));
+        assertThat(recovered.isMessageDeleted(positions.get(1))).isFalse();
+        assertThat(recovered.isMessageDeleted(positions.get(2))).isTrue();
+        assertThat(recovered.getBatchPositionAckSet(positions.get(4))).containsExactly(ackSet);
+    }
+
+    @Test(dataProvider = "booleans")
+    public void testMetadataStoreRecoveryHandlesMissingOrDisabledBatchIndexAck(boolean recoverBatchIndexAck)
+            throws Exception {
+        ManagedLedgerConfig writeConfig = new ManagedLedgerConfig()
+                .setMaxUnackedRangesToPersistInMetadataStore(20);
+        writeConfig.setDeletionAtBatchIndexLevelEnabled(!recoverBatchIndexAck);
+        String name = "tenant/ns/persistent/batch-metadata-compatibility-" + recoverBatchIndexAck;
+        @Cleanup
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(name, writeConfig);
+        ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub");
+        Position position = ledger.addEntry(new byte[] {1});
+        if (!recoverBatchIndexAck) {
+            cursor.delete(AckSetStateUtil.createPositionWithAckSet(
+                    position.getLedgerId(), position.getEntryId(), new long[] {2L}));
+        }
+        ledger.close();
+
+        ManagedCursorInfo persisted = storedCursorInfo(ledger);
+        assertThat(persisted.getCursorsLedgerId()).isEqualTo(-1L);
+        assertThat(persisted.getBatchedEntryDeletionIndexInfosCount())
+                .isEqualTo(recoverBatchIndexAck ? 0 : 1);
+
+        ManagedLedgerConfig recoveryConfig = new ManagedLedgerConfig()
+                .setMaxUnackedRangesToPersistInMetadataStore(20);
+        recoveryConfig.setDeletionAtBatchIndexLevelEnabled(recoverBatchIndexAck);
+        @Cleanup
+        ManagedLedgerImpl reopened = (ManagedLedgerImpl) factory.open(name, recoveryConfig);
+        ManagedCursorImpl recovered = (ManagedCursorImpl) reopened.openCursor("sub");
+        assertThat(recovered.getBatchPositionAckSet(position)).isNull();
+    }
+
+    private static long[] deleteSet(BatchedEntryDeletionIndexInfo indexInfo) {
+        long[] result = new long[indexInfo.getDeleteSetsCount()];
+        for (int i = 0; i < result.length; i++) {
+            result[i] = indexInfo.getDeleteSetAt(i);
+        }
+        return result;
+    }
+
+    private static ManagedCursorInfo storedCursorInfo(ManagedLedgerImpl ledger) throws Exception {
+        CompletableFuture<ManagedCursorInfo> result = new CompletableFuture<>();
+        ledger.getStore().asyncGetCursorInfo(ledger.getName(), "sub", new MetaStoreCallback<>() {
+            @Override
+            public void operationComplete(ManagedCursorInfo info, Stat stat) {
+                result.complete(info);
+            }
+
+            @Override
+            public void operationFailed(MetaStoreException exception) {
+                result.completeExceptionally(exception);
+            }
+        });
+        return result.get(5, TimeUnit.SECONDS);
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckMetadataStoreRecoveryTest.java b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckMetadataStoreRecoveryTest.java
new file mode 100644
index 000000000000..cfb3ba32290d
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorBatchAckMetadataStoreRecoveryTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
+import org.apache.bookkeeper.mledger.proto.ManagedCursorInfo;
+import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.MessageIdAdv;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.metadata.api.Stat;
+import org.awaitility.Awaitility;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-impl")
+public class ManagedCursorBatchAckMetadataStoreRecoveryTest extends SharedPulsarBaseTest {
+
+    @Test(timeOut = 30000)
+    public void testSharedConsumerSkipsPartiallyAcknowledgedBatchAfterTopicUnload() throws Exception {
+        int batchSize = 4;
+        String topicName = newTopicName();
+        String subscriptionName = "shared-sub";
+        List<MessageId> sentMessageIds = new ArrayList<>(batchSize);
+
+        try (Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
+                .topic(topicName)
+                .enableBatching(true)
+                .batchingMaxMessages(batchSize)
+                .batchingMaxPublishDelay(1, TimeUnit.HOURS)
+                .create();
+             Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
+                     .topic(topicName)
+                     .subscriptionName(subscriptionName)
+                     .subscriptionType(SubscriptionType.Shared)
+                     .receiverQueueSize(batchSize)
+                     .enableBatchIndexAcknowledgment(true)
+                     .isAckReceiptEnabled(true)
+                     .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS)
+                     .subscribe()) {
+            List<CompletableFuture<MessageId>> sendFutures = new ArrayList<>(batchSize);
+            for (int i = 0; i < batchSize; i++) {
+                sendFutures.add(producer.sendAsync(i));
+            }
+            producer.flush();
+            for (CompletableFuture<MessageId> sendFuture : sendFutures) {
+                sentMessageIds.add(sendFuture.get(5, TimeUnit.SECONDS));
+            }
+            assertSingleBatch(sentMessageIds, batchSize);
+
+            List<Message<Integer>> messages = new ArrayList<>(batchSize);
+            try {
+                for (int i = 0; i < batchSize; i++) {
+                    Message<Integer> message = consumer.receive(5, TimeUnit.SECONDS);
+                    assertThat(message).as("message %s from the original batch", i).isNotNull();
+                    assertThat(message.getValue()).isEqualTo(i);
+                    messages.add(message);
+                }
+                consumer.acknowledge(messages.get(0));
+                consumer.acknowledge(messages.get(1));
+            } finally {
+                messages.forEach(Message::release);
+            }
+        }
+
+        PersistentTopic originalTopic = (PersistentTopic) getTopicReference(topicName).orElseThrow();
+        ManagedLedgerImpl originalLedger = (ManagedLedgerImpl) originalTopic.getManagedLedger();
+        ManagedCursorImpl originalCursor = cursor(originalTopic, subscriptionName);
+        MessageIdAdv batchMessageId = (MessageIdAdv) sentMessageIds.get(0);
+        Position batchPosition = PositionFactory.create(batchMessageId.getLedgerId(), batchMessageId.getEntryId());
+        Awaitility.await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+                assertThat(originalCursor.getBatchPositionAckSet(batchPosition)).isNotNull());
+        long[] ackSetBeforeUnload = originalCursor.getBatchPositionAckSet(batchPosition);
+
+        admin.topics().unload(topicName);
+        Awaitility.await().atMost(Duration.ofSeconds(5))
+                .until(() -> getTopicReference(topicName).isEmpty());
+
+        ManagedCursorInfo persisted = storedCursorInfo(originalLedger, subscriptionName);
+        assertThat(persisted.getCursorsLedgerId()).isEqualTo(-1L);
+        assertThat(persisted.getBatchedEntryDeletionIndexInfosCount())
+                .as("the partial batch ACK was written to MetadataStore before recovery")
+                .isEqualTo(1);
+        assertThat(persisted.getBatchedEntryDeletionIndexInfoAt(0).getPosition().getLedgerId())
+                .isEqualTo(batchPosition.getLedgerId());
+        assertThat(persisted.getBatchedEntryDeletionIndexInfoAt(0).getPosition().getEntryId())
+                .isEqualTo(batchPosition.getEntryId());
+
+        try (PulsarClient newClient = newPulsarClient();
+             Consumer<Integer> recoveredConsumer = newClient.newConsumer(Schema.INT32)
+                     .topic(topicName)
+                     .subscriptionName(subscriptionName)
+                     .subscriptionType(SubscriptionType.Shared)
+                     .receiverQueueSize(batchSize)
+                     .enableBatchIndexAcknowledgment(true)
+                     .isAckReceiptEnabled(true)
+                     .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS)
+                     .subscribe()) {
+            PersistentTopic recoveredTopic = (PersistentTopic) getTopicReference(topicName).orElseThrow();
+            ManagedCursorImpl recoveredCursor = cursor(recoveredTopic, subscriptionName);
+            assertThat(recoveredCursor).as("topic unload must rebuild the cursor").isNotSameAs(originalCursor);
+            assertThat(recoveredCursor.getBatchPositionAckSet(batchPosition)).containsExactly(ackSetBeforeUnload);
+
+            Set<Integer> expectedValues = new HashSet<>(Set.of(2, 3));
+            for (int i = 0; i < 2; i++) {
+                Message<Integer> message = recoveredConsumer.receive(5, TimeUnit.SECONDS);
+                assertThat(message).as("unacknowledged message %s", i).isNotNull();
+                try {
+                    assertThat(expectedValues.remove(message.getValue()))
+                            .as("only unacknowledged messages should be delivered")
+                            .isTrue();
+                    recoveredConsumer.acknowledge(message);
+                } finally {
+                    message.release();
+                }
+            }
+            assertThat(expectedValues).isEmpty();
+
+            Message<Integer> unexpected = recoveredConsumer.receive(1, TimeUnit.SECONDS);
+            try {
+                assertThat(unexpected).as("acknowledged batch indexes must not be redelivered").isNull();
+            } finally {
+                if (unexpected != null) {
+                    unexpected.release();
+                }
+            }
+        }
+    }
+
+    private static void assertSingleBatch(List<MessageId> messageIds, int batchSize) {
+        assertThat(messageIds).hasSize(batchSize);
+        MessageIdAdv first = (MessageIdAdv) messageIds.get(0);
+        for (int i = 0; i < messageIds.size(); i++) {
+            MessageIdAdv current = (MessageIdAdv) messageIds.get(i);
+            assertThat(current.getLedgerId()).isEqualTo(first.getLedgerId());
+            assertThat(current.getEntryId()).isEqualTo(first.getEntryId());
+            assertThat(current.getBatchIndex()).isEqualTo(i);
+        }
+    }
+
+    private static ManagedCursorImpl cursor(PersistentTopic topic, String subscriptionName) {
+        PersistentSubscription subscription = topic.getSubscription(subscriptionName);
+        return (ManagedCursorImpl) subscription.getCursor();
+    }
+
+    private static ManagedCursorInfo storedCursorInfo(ManagedLedgerImpl ledger, String subscriptionName)
+            throws Exception {
+        CompletableFuture<ManagedCursorInfo> result = new CompletableFuture<>();
+        ledger.getStore().asyncGetCursorInfo(ledger.getName(), subscriptionName, new MetaStoreCallback<>() {
+            @Override
+            public void operationComplete(ManagedCursorInfo info, Stat stat) {
+                result.complete(info);
+            }
+
+            @Override
+            public void operationFailed(MetaStoreException exception) {
+                result.completeExceptionally(exception);
+            }
+        });
+        return result.get(5, TimeUnit.SECONDS);
+    }
+}


(pulsar) 27/27: [fix][broker] Preserve compaction state on ledger close failure (#26665)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 8fd3aa10aae49ebc82d27bf0b10a52456825890d
Author: sinan liu <li...@gmail.com>
AuthorDate: Tue Sep 22 14:19:09 2026 +0800

    [fix][broker] Preserve compaction state on ledger close failure (#26665)
    
    (cherry picked from commit b642e8e883b2c04fa9881f4d43c4214d21b644e9)
---
 .../compaction/StrategicTwoPhaseCompactor.java     |   4 +-
 .../broker/service/SharedPulsarBaseTest.java       |   7 ++
 .../persistent/CompactionConcurrencyTest.java      | 101 +++++++++++++++++++++
 3 files changed, 111 insertions(+), 1 deletion(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
index c051e48c422c..77f0a2c6ae46 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
@@ -359,6 +359,9 @@ public class StrategicTwoPhaseCompactor extends PublishingOrderCompactor {
                                 }
                             });
                 })
+                // Close before publishing the ledger: a close failure must not delete a ledger
+                // already referenced by the compaction subscription.
+                .thenCompose(v -> closeLedger(ledger))
                 .thenCompose(v -> {
                     log.info("Acking ledger id {}", phaseOneResult.lastId);
                     return ((CompactionReaderImpl<T>) reader)
@@ -366,7 +369,6 @@ public class StrategicTwoPhaseCompactor extends PublishingOrderCompactor {
                                     phaseOneResult.lastId, Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY,
                                             ledger.getId()));
                 })
-                .thenCompose((v) -> closeLedger(ledger))
                 .whenComplete((v, exception) -> {
                     if (exception != null) {
                         deleteLedger(bk, ledger).whenComplete((res2, exception2) -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarBaseTest.java
index 17d2fdbfaf25..a724f60f3bee 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarBaseTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SharedPulsarBaseTest.java
@@ -82,6 +82,13 @@ public abstract class SharedPulsarBaseTest {
         return PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build();
     }
 
+    /**
+     * The shared {@link PulsarService}, for tests that need broker-internal accessors.
+     */
+    protected PulsarService getPulsar() {
+        return pulsar;
+    }
+
     /**
      * Returns the shared broker's {@link ServiceConfiguration} for runtime config inspection or changes.
      */
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/CompactionConcurrencyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/CompactionConcurrencyTest.java
index b4f8a0807c02..df40bd9aa912 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/CompactionConcurrencyTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/CompactionConcurrencyTest.java
@@ -18,21 +18,122 @@
  */
 package org.apache.pulsar.broker.service.persistent;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.LedgerMetadataBuilder;
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.common.topics.TopicCompactionStrategy;
 import org.apache.pulsar.compaction.Compactor;
+import org.apache.pulsar.compaction.PulsarTopicCompactionService;
+import org.apache.pulsar.compaction.StrategicTwoPhaseCompactor;
 import org.awaitility.Awaitility;
 import org.testng.annotations.Test;
 
 @Test(groups = "broker")
 public class CompactionConcurrencyTest extends SharedPulsarBaseTest {
 
+    @Test(timeOut = 60000)
+    public void testStrategicCompactionCloseFailurePreservesPublishedLedger() throws Exception {
+        String topicName = newTopicName();
+        BookKeeper bookKeeper = getPulsar().getBookKeeperClient();
+        var strategy = new TopicCompactionStrategy<String>() {
+            @Override
+            public Schema<String> getSchema() {
+                return Schema.STRING;
+            }
+
+            @Override
+            public boolean shouldKeepLeft(String previous, String current) {
+                return false;
+            }
+        };
+        var compactor = new StrategicTwoPhaseCompactor(getConfig(), pulsarClient, bookKeeper,
+                getPulsar().getCompactorExecutor());
+        try (var producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create()) {
+            producer.newMessage().key("key").value("original").send();
+            var originalCompaction = compactor.compact(topicName, strategy);
+            assertThat(originalCompaction).succeedsWithin(15, TimeUnit.SECONDS);
+            long originalLedgerId = originalCompaction.join();
+            var topicFuture = getTopic(topicName, false);
+            assertThat(topicFuture).succeedsWithin(10, TimeUnit.SECONDS);
+            PersistentTopic topic = (PersistentTopic) topicFuture.join().orElseThrow();
+            var cursor = topic.getSubscription(Compactor.COMPACTION_SUBSCRIPTION).getCursor();
+            Awaitility.await().untilAsserted(() -> assertThat(cursor.getProperties())
+                    .containsEntry(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, originalLedgerId));
+            Position originalPosition = cursor.getMarkDeletedPosition();
+            producer.newMessage().key("key").value("updated").send();
+
+            AtomicLong failedLedgerId = new AtomicLong(-1);
+            AtomicReference<Throwable> closeFailure = new AtomicReference<>();
+            var failingCompactor = new StrategicTwoPhaseCompactor(getConfig(), pulsarClient, bookKeeper,
+                    getPulsar().getCompactorExecutor()) {
+                @Override
+                protected CompletableFuture<Void> closeLedger(LedgerHandle ledger) {
+                    failedLedgerId.set(ledger.getId());
+                    // Persist a conflicting close through the real metadata manager. BookKeeper's own
+                    // close path must detect the inconsistent length and report MetadataVersionException.
+                    var ledgerManager = bookKeeper.getLedgerManager();
+                    return ledgerManager.readLedgerMetadata(ledger.getId()).thenCompose(metadata ->
+                            ledgerManager.writeLedgerMetadata(ledger.getId(),
+                                    LedgerMetadataBuilder.from(metadata.getValue()).withClosedState()
+                                            .withLastEntryId(ledger.getLastAddConfirmed())
+                                            .withLength(ledger.getLength() + 1).build(), metadata.getVersion()))
+                            .thenCompose(__ -> super.closeLedger(ledger)
+                                    .whenComplete((ignored, error) -> closeFailure.set(error)));
+                }
+            };
+            assertThat(failingCompactor.compact(topicName, strategy))
+                    .failsWithin(15, TimeUnit.SECONDS)
+                    .withThrowableThat()
+                    .withRootCauseInstanceOf(BKException.BKMetadataVersionException.class);
+            assertThat(closeFailure.get()).isInstanceOf(BKException.BKMetadataVersionException.class);
+
+            assertThat(cursor.getProperties())
+                    .containsEntry(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, originalLedgerId);
+            assertThat(cursor.getMarkDeletedPosition()).isEqualTo(originalPosition);
+            var compactedTopic = ((PulsarTopicCompactionService) topic.getTopicCompactionService()).getCompactedTopic();
+            assertThat(compactedTopic.getCompactedTopicContextFuture())
+                    .succeedsWithin(10, TimeUnit.SECONDS)
+                    .extracting(context -> context.getLedger().getId())
+                    .isEqualTo(originalLedgerId);
+            assertThat(bookKeeper.getLedgerManager().readLedgerMetadata(originalLedgerId))
+                    .succeedsWithin(10, TimeUnit.SECONDS)
+                    .satisfies(metadata -> assertThat(metadata.getValue().isClosed()).isTrue());
+            assertThat(bookKeeper.getLedgerManager().readLedgerMetadata(failedLedgerId.get()))
+                    .failsWithin(10, TimeUnit.SECONDS)
+                    .withThrowableThat()
+                    .withRootCauseInstanceOf(BKException.BKNoSuchLedgerExistsOnMetadataServerException.class);
+
+            // A normal retry must publish a closed ledger containing the updated value.
+            var retryCompaction = compactor.compact(topicName, strategy);
+            assertThat(retryCompaction).succeedsWithin(15, TimeUnit.SECONDS);
+            long retryLedgerId = retryCompaction.join();
+            Awaitility.await().untilAsserted(() -> assertThat(cursor.getProperties())
+                    .containsEntry(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, retryLedgerId));
+            assertThat(bookKeeper.getLedgerManager().readLedgerMetadata(retryLedgerId))
+                    .succeedsWithin(10, TimeUnit.SECONDS)
+                    .satisfies(metadata -> assertThat(metadata.getValue().isClosed()).isTrue());
+            try (var reader = pulsarClient.newReader(Schema.STRING).topic(topicName)
+                    .startMessageId(MessageId.earliest).readCompacted(true).create()) {
+                var message = reader.readNext(10, TimeUnit.SECONDS);
+                assertThat(message).isNotNull();
+                assertThat(message.getKey()).isEqualTo("key");
+                assertThat(message.getValue()).isEqualTo("updated");
+            }
+        }
+    }
+
     private void triggerCompactionAndWait(String topicName) throws Exception {
         PersistentTopic persistentTopic =
                 (PersistentTopic) getTopic(topicName, false).get().get();


(pulsar) 25/27: [fix][broker] Apply the role logging anonymizer to topic authorization denial logs (#26642)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 304a8729494dd6e6bbe1ca7abfc1042af627a2af
Author: kannar <ka...@gmail.com>
AuthorDate: Sat Sep 19 17:04:59 2026 +0200

    [fix][broker] Apply the role logging anonymizer to topic authorization denial logs (#26642)
    
    Apply the configured role logging anonymizer consistently to topic authorization denial logs.
    
    (cherry picked from commit 03174f696d4b3d3cc80d1f5baf90169856bc6636)
---
 ...DefaultAuthenticationRoleLoggingAnonymizer.java |  3 +-
 ...ultAuthenticationRoleLoggingAnonymizerTest.java | 42 ++++++++++++++++++++++
 .../apache/pulsar/broker/service/ServerCnx.java    |  3 +-
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizer.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizer.java
index 151118c6d5e4..9f94230e7818 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizer.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizer.java
@@ -62,6 +62,7 @@ public final class DefaultAuthenticationRoleLoggingAnonymizer {
    }
 
    public String anonymize(String role) {
-      return anonymizerType.anonymize(role);
+      // originalPrincipal is null for clients that do not connect through a proxy
+      return role == null ? null : anonymizerType.anonymize(role);
    }
 }
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizerTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizerTest.java
new file mode 100644
index 000000000000..1adaa597638f
--- /dev/null
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/anonymizer/DefaultAuthenticationRoleLoggingAnonymizerTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.configuration.anonymizer;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
+import org.testng.annotations.Test;
+
+public class DefaultAuthenticationRoleLoggingAnonymizerTest {
+
+    @Test
+    public void testNullRoleIsKeptForEveryType() {
+        // originalPrincipal is null for clients that do not connect through a proxy
+        for (DefaultRoleAnonymizerType type : DefaultRoleAnonymizerType.values()) {
+            assertNull(new DefaultAuthenticationRoleLoggingAnonymizer(type.name()).anonymize(null), type.name());
+        }
+    }
+
+    @Test
+    public void testRoleIsAnonymized() {
+        assertEquals(new DefaultAuthenticationRoleLoggingAnonymizer("NONE").anonymize("role"), "role");
+        assertEquals(new DefaultAuthenticationRoleLoggingAnonymizer("REDACTED").anonymize("role"), "[REDACTED]");
+        assertEquals(new DefaultAuthenticationRoleLoggingAnonymizer("SHA256").anonymize("role").substring(0, 8),
+                "SHA-256:");
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 852494e58e0b..185b2ad2b5ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -569,7 +569,8 @@ public class ServerCnx extends PulsarHandler implements TransportCnx {
         result.thenAccept(isAuthorized -> {
             if (!isAuthorized) {
                 log.warn("Role {} or OriginalRole {} is not authorized to perform operation {} on topic {}",
-                        authRole, originalPrincipal, operation, topicName);
+                        authenticationRoleLoggingAnonymizer.anonymize(authRole),
+                        authenticationRoleLoggingAnonymizer.anonymize(originalPrincipal), operation, topicName);
             }
         });
         return result;


(pulsar) 21/27: [improve][broker] Reduce cache retention after reads and apply retention settings at startup (#26627)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 80df0506ebf3d50068d2374065ab58af4e8eda8f
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 17 19:42:58 2026 +0300

    [improve][broker] Reduce cache retention after reads and apply retention settings at startup (#26627)
    
    (cherry picked from commit bed06a0a9c5731c0395e73908212842731d8b255)
---
 conf/broker.conf                                   |  4 +-
 conf/standalone.conf                               |  4 +-
 .../apache/pulsar/broker/ServiceConfiguration.java |  4 +-
 .../pulsar/broker/ManagedLedgerClientFactory.java  |  4 ++
 .../broker/ManagedLedgerClientFactoryTest.java     | 75 ++++++++++++++++++++++
 5 files changed, 85 insertions(+), 6 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index fbd556405941..cec9560ed2a3 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1344,8 +1344,8 @@ managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes=5
 # When disabled:
 # - Cache behaves more like a FIFO queue with time-based and size-based eviction
 # - Minimum eviction time is managedLedgerCacheEvictionTimeThresholdMillis
-# Default is true, to behave like a LRU cache.
-managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed=true
+# Default is false, to avoid extending cache retention for entries that have already been read.
+managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed=false
 
 # This setting configures the duration of continuing to cache added entries while there are no
 # active cursors, when the last active cursor has left or immediately after initialization when
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 6c1e94c953c6..19240fbb6601 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -876,8 +876,8 @@ managedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes=5
 # When disabled:
 # - Cache behaves more like a FIFO queue with time-based and size-based eviction
 # - Minimum eviction time is managedLedgerCacheEvictionTimeThresholdMillis
-# Default is true, to behave like a LRU cache.
-managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed=true
+# Default is false, to avoid extending cache retention for entries that have already been read.
+managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed=false
 
 # This setting configures the duration of continuing to cache added entries while there are no
 # active cursors, when the last active cursor has left or immediately after initialization when
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 8093ce77d0e0..38d9877611f5 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2376,9 +2376,9 @@ public class ServiceConfiguration implements PulsarConfiguration {
                     + "When disabled:\n"
                     + " - Cache behaves more like a FIFO queue with time-based and size-based eviction\n"
                     + " - Minimum eviction time is managedLedgerCacheEvictionTimeThresholdMillis\n"
-                    + "Default is true, to behave like a LRU cache."
+                    + "Default is false, to avoid extending cache retention for entries that have already been read."
     )
-    private boolean managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed = true;
+    private boolean managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed = false;
 
     @FieldContext(category = CATEGORY_STORAGE_ML,
             doc = "Configure the threshold (in number of entries) from where a cursor should be considered 'backlogged'"
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
index cc16668790dd..69b59596ab19 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
@@ -78,6 +78,10 @@ public class ManagedLedgerClientFactory implements ManagedLedgerStorage {
         managedLedgerFactoryConfig.setCacheEvictionIntervalMs(conf.getManagedLedgerCacheEvictionIntervalMs());
         managedLedgerFactoryConfig.setCacheEvictionTimeThresholdMillis(
                 conf.getManagedLedgerCacheEvictionTimeThresholdMillis());
+        managedLedgerFactoryConfig.setCacheEvictionExtendTTLOfRecentlyAccessed(
+                conf.isManagedLedgerCacheEvictionExtendTTLOfRecentlyAccessed());
+        managedLedgerFactoryConfig.setCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes(
+                conf.getManagedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes());
         Long continueCachingAddedEntriesAfterLastActiveCursorLeavesMillis =
                 conf.getManagedLedgerContinueCachingAddedEntriesAfterLastActiveCursorLeavesMillis();
         if (continueCachingAddedEntriesAfterLastActiveCursorLeavesMillis != null) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/ManagedLedgerClientFactoryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/ManagedLedgerClientFactoryTest.java
new file mode 100644
index 000000000000..5b6c7d9cd0c2
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/ManagedLedgerClientFactoryTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import io.netty.channel.EventLoopGroup;
+import io.opentelemetry.api.OpenTelemetry;
+import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
+import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.mockito.ArgumentCaptor;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class ManagedLedgerClientFactoryTest {
+    @DataProvider
+    public Object[][] cacheExtensionSettings() {
+        return new Object[][] {{null, 5}, {false, 2}, {true, 0}, {true, 5}};
+    }
+
+    @Test(dataProvider = "cacheExtensionSettings")
+    public void testCacheExtensionSettingsAtStartup(Boolean extendRecentlyAccessed, int maxExtensions)
+            throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        if (extendRecentlyAccessed == null) {
+            assertThat(conf.isManagedLedgerCacheEvictionExtendTTLOfRecentlyAccessed()).isFalse();
+        } else {
+            conf.setManagedLedgerCacheEvictionExtendTTLOfRecentlyAccessed(extendRecentlyAccessed);
+        }
+        conf.setManagedLedgerCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes(maxExtensions);
+        conf.setBookkeeperClientExposeStatsToPrometheus(false);
+        BookKeeperClientFactory bookkeeperProvider = mock(BookKeeperClientFactory.class);
+        when(bookkeeperProvider.create(any(), any(), any(), any(), isNull(), any()))
+                .thenReturn(CompletableFuture.completedFuture(mock(BookKeeper.class)));
+        try (ManagedLedgerClientFactory factory = spy(new ManagedLedgerClientFactory())) {
+            doReturn(mock(ManagedLedgerFactoryImpl.class)).when(factory)
+                    .createManagedLedgerFactory(any(), any(), any(), any(), any());
+            factory.initialize(conf, mock(MetadataStoreExtended.class), bookkeeperProvider,
+                    mock(EventLoopGroup.class), OpenTelemetry.noop());
+            ArgumentCaptor<ManagedLedgerFactoryConfig> config =
+                    ArgumentCaptor.forClass(ManagedLedgerFactoryConfig.class);
+            verify(factory).createManagedLedgerFactory(any(), any(), any(), config.capture(), any());
+            assertThat(config.getValue().isCacheEvictionExtendTTLOfRecentlyAccessed())
+                    .isEqualTo(Boolean.TRUE.equals(extendRecentlyAccessed));
+            assertThat(config.getValue().getCacheEvictionExtendTTLOfEntriesWithRemainingExpectedReadsMaxTimes())
+                    .isEqualTo(maxExtensions);
+        }
+    }
+}


(pulsar) 24/27: [fix][test] Isolate token CLI test output from background logging (#26657)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 71884df099c53c081749b9ec4851e18c282f1d7a
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Sat Sep 19 18:01:08 2026 +0300

    [fix][test] Isolate token CLI test output from background logging (#26657)
    
    (cherry picked from commit d921fc44e56b185a91c920237875b9982864744b)
---
 .../utils/auth/tokens/TokensCliUtilsTest.java      | 43 +++++++++++++++++++---
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java
index aae379dae147..124a12f633c9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtilsTest.java
@@ -26,6 +26,7 @@ import io.jsonwebtoken.JwsHeader;
 import io.jsonwebtoken.Jwts;
 import io.jsonwebtoken.io.Decoders;
 import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
 import java.io.PrintStream;
 import java.lang.reflect.Field;
 import java.time.Instant;
@@ -56,9 +57,10 @@ public class TokensCliUtilsTest {
     @Test
     public void testCreateToken() {
         PrintStream oldStream = System.out;
+        ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
+        PrintStream capturedStream = captureOutputForCurrentThread(baoStream, oldStream);
         try {
-            ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
-            System.setOut(new PrintStream(baoStream));
+            System.setOut(capturedStream);
 
             new TokensCliUtils().execute(new String[]{"create-secret-key", "--base64"});
             String secretKey = baoStream.toString();
@@ -89,6 +91,7 @@ public class TokensCliUtilsTest {
             throw new RuntimeException(e);
         } finally {
             System.setOut(oldStream);
+            capturedStream.close();
         }
     }
 
@@ -96,10 +99,11 @@ public class TokensCliUtilsTest {
     public void commandCreateToken_WhenCreatingATokenWithExpiryTime_ShouldHaveTheDesiredExpireTime(String expireTime,
                                                                                 int expireAsSec) throws Exception {
         PrintStream oldStream = System.out;
+        ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
+        PrintStream capturedStream = captureOutputForCurrentThread(baoStream, oldStream);
         try {
             //Arrange
-            ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
-            System.setOut(new PrintStream(baoStream));
+            System.setOut(capturedStream);
 
             String[] command = {"create", "--secret-key",
                     "data:;base64,u+FxaxYWpsTfxeEmMh8fQeS3g2jfXw4+sGIv+PTY+BY=",
@@ -129,6 +133,7 @@ public class TokensCliUtilsTest {
             throw new RuntimeException(e);
         } finally {
             System.setOut(oldStream);
+            capturedStream.close();
         }
     }
 
@@ -140,9 +145,10 @@ public class TokensCliUtilsTest {
     @Test
     public void testGenerateDocs() throws Exception {
         PrintStream oldStream = System.out;
+        ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
+        PrintStream capturedStream = captureOutputForCurrentThread(baoStream, oldStream);
         try {
-            ByteArrayOutputStream baoStream = new ByteArrayOutputStream();
-            System.setOut(new PrintStream(baoStream));
+            System.setOut(capturedStream);
 
             new TokensCliUtils().execute(new String[]{"gen-doc"});
 
@@ -162,9 +168,34 @@ public class TokensCliUtilsTest {
 
         } finally {
             System.setOut(oldStream);
+            capturedStream.close();
         }
     }
 
+    private static PrintStream captureOutputForCurrentThread(ByteArrayOutputStream captured, PrintStream original) {
+        Thread testThread = Thread.currentThread();
+        // Shared brokers can still log while the CLI runs. Keep their output out of the captured token.
+        return new PrintStream(new OutputStream() {
+            @Override
+            public void write(int b) {
+                if (Thread.currentThread() == testThread) {
+                    captured.write(b);
+                } else {
+                    original.write(b);
+                }
+            }
+
+            @Override
+            public void write(byte[] b, int off, int len) {
+                if (Thread.currentThread() == testThread) {
+                    captured.write(b, off, len);
+                } else {
+                    original.write(b, off, len);
+                }
+            }
+        });
+    }
+
     private void assertInnerClass(String className, String message) throws Exception {
         Class argumentsClass = Class.forName(className);
         Field[] fields = argumentsClass.getDeclaredFields();


(pulsar) 20/27: [fix][client] Drop a send receipt for a removed producer, not the connection (#26618)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit d63678a4d9dd9210f81c2e343db80895d9f88e66
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Wed Sep 16 22:46:29 2026 -0700

    [fix][client] Drop a send receipt for a removed producer, not the connection (#26618)
    
    (cherry picked from commit 4315708040d9b5113d652359db724c2bf55c58ea)
---
 .../org/apache/pulsar/client/impl/ClientCnx.java   |  2 +-
 .../apache/pulsar/client/impl/ClientCnxTest.java   | 27 ++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
index dc73f435c7f4..ee97e66df488 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
@@ -502,7 +502,7 @@ public class ClientCnx extends PulsarHandler {
                 producer.printWarnLogWhenCanNotDetermineDeduplication(ctx.channel(), sequenceId, highestSequenceId);
             }
 
-        } else {
+        } else if (producer != null) {
             if (log.isDebugEnabled()) {
                 log.debug("{} Got receipt for producer: [id:{}, name:{}] -- sequence-id: {}-{} -- entry-id: {}:{}",
                         ctx.channel(), producerId, producer.getProducerName(), sequenceId, highestSequenceId,
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java
index 20de6eacf55b..2bd862e343d6 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.common.api.proto.CommandCloseProducer;
 import org.apache.pulsar.common.api.proto.CommandConnected;
 import org.apache.pulsar.common.api.proto.CommandError;
 import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse;
+import org.apache.pulsar.common.api.proto.CommandSendReceipt;
 import org.apache.pulsar.common.api.proto.CommandWatchTopicListSuccess;
 import org.apache.pulsar.common.api.proto.CommandWatchTopicUpdate;
 import org.apache.pulsar.common.api.proto.ServerError;
@@ -292,6 +293,32 @@ public class ClientCnxTest {
         eventLoop.shutdownGracefully();
     }
 
+    @Test
+    public void testSendReceiptForRemovedProducerIsIgnored() {
+        ThreadFactory threadFactory = new DefaultThreadFactory("testSendReceiptForRemovedProducerIsIgnored");
+        EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory);
+        ClientConfigurationData conf = new ClientConfigurationData();
+        ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop);
+        cnx.state = ClientCnx.State.Ready;
+
+        // A receipt for a producer the broker has already told the client to close, as happens when
+        // its topic is terminated with sends in flight, is dropped rather than failing the connection.
+        long producerId = 1;
+        CommandSendReceipt receipt = new CommandSendReceipt()
+                .setProducerId(producerId)
+                .setSequenceId(5)
+                .setHighestSequenceId(5);
+        receipt.setMessageId().setLedgerId(3).setEntryId(7);
+        cnx.handleSendReceipt(receipt);
+
+        ProducerImpl<?> producer = mock(ProducerImpl.class);
+        cnx.registerProducer(producerId, producer);
+        cnx.handleSendReceipt(receipt);
+        verify(producer).ackReceived(cnx, 5, 5, 3, 7);
+
+        eventLoop.shutdownGracefully();
+    }
+
     @Test
     public void testIdleCheckWithTopicListWatcher() {
         ClientCnx cnx =


(pulsar) 01/27: [improve][broker] Disable subscription-thread dispatch by default (#26578)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit b8620060f9c2bd08d80153ea3cc7a0e0b538d003
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Sep 14 22:59:57 2026 +0300

    [improve][broker] Disable subscription-thread dispatch by default (#26578)
    
    (cherry picked from commit c613d254c271154530a3fb5ebe4fdecf540fbef9)
---
 conf/broker.conf                                                        | 2 +-
 .../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 7d368882735a..fbd556405941 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -508,7 +508,7 @@ dispatchThrottlingOnNonBacklogConsumerEnabled=true
 dispatcherMaxReadBatchSize=100
 
 # Dispatch messages and execute broker side filters in a per-subscription thread
-dispatcherDispatchMessagesInSubscriptionThread=true
+dispatcherDispatchMessagesInSubscriptionThread=false
 
 # Max size in bytes of entries to read from bookkeeper. By default it is 5MB.
 dispatcherMaxReadSizeBytes=5242880
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ccf62c04b4b2..8093ce77d0e0 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1336,7 +1336,7 @@ public class ServiceConfiguration implements PulsarConfiguration {
             category = CATEGORY_SERVER,
             doc = "Dispatch messages and execute broker side filters in a per-subscription thread"
     )
-    private boolean dispatcherDispatchMessagesInSubscriptionThread = true;
+    private boolean dispatcherDispatchMessagesInSubscriptionThread = false;
 
     @FieldContext(
         dynamic = false,


(pulsar) 17/27: [fix][test] Avoid blocking common-pool workers in ledger shutdown test (#26560)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit fbad85453fedee5d4e5f687e37457f3bfc6ac243
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Sun Sep 13 18:15:12 2026 +0300

    [fix][test] Avoid blocking common-pool workers in ledger shutdown test (#26560)
    
    (cherry picked from commit db51d0d55ae96b563e2de3cdae7a2172e56905d0)
---
 .../impl/ManagedLedgerFactoryShutdownTest.java      | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java
index df1cf82d9d42..0572ad44167a 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryShutdownTest.java
@@ -57,7 +57,7 @@ import org.testng.annotations.Test;
 public class ManagedLedgerFactoryShutdownTest {
 
     private final String ledgerName = UUID.randomUUID().toString();
-    private final CountDownLatch slowZk = new CountDownLatch(1);
+    private CompletableFuture<Void> metadataReadGate;
 
     private MetadataStoreExtended metadataStore;
     private BookKeeper bookKeeper;
@@ -68,6 +68,7 @@ public class ManagedLedgerFactoryShutdownTest {
         final long version = 0;
         final long createTimeMillis = System.currentTimeMillis();
 
+        metadataReadGate = new CompletableFuture<>();
         metadataStore = mock(MetadataStoreExtended.class);
         bookKeeper = mock(BookKeeper.class);
 
@@ -84,12 +85,7 @@ public class ManagedLedgerFactoryShutdownTest {
                         .setEntries(0)
                         .setTimestamp(System.currentTimeMillis());
                 Stat stat = new Stat(path, version, createTimeMillis, createTimeMillis, false, false);
-                return CompletableFuture.supplyAsync(() -> {
-                    try {
-                        slowZk.await();
-                    } catch (InterruptedException e) {
-                        e.printStackTrace();
-                    }
+                return metadataReadGate.thenApplyAsync(__ -> {
                     log.info("metadataStore.get({}) returned,managedLedgerInfo={},stat={}", path, mli,
                             stat);
                     return Optional.of(new GetResult(mli.toByteArray(), stat));
@@ -101,12 +97,7 @@ public class ManagedLedgerFactoryShutdownTest {
                         .setMarkDeleteLedgerId(0)
                         .setMarkDeleteLedgerId(-1);
                 Stat stat = new Stat(path, version, createTimeMillis, createTimeMillis, false, false);
-                return CompletableFuture.supplyAsync(() -> {
-                    try {
-                        slowZk.await();
-                    } catch (InterruptedException e) {
-                        e.printStackTrace();
-                    }
+                return metadataReadGate.thenApplyAsync(__ -> {
                     log.info("metadataStore.get({}) returned:managedCursorInfo={},stat={}", path, mci,
                             stat);
                     return Optional.of(new GetResult(mci.toByteArray(), stat));
@@ -188,8 +179,8 @@ public class ManagedLedgerFactoryShutdownTest {
 
 
         factory.shutdownAsync().get();
-        //make zk returned after factory shutdown
-        slowZk.countDown();
+        // Complete delayed metadata reads only after shutdown, without blocking common-pool workers.
+        metadataReadGate.complete(null);
 
         //
         Assert.assertTrue(callbackInvoked.await(5, TimeUnit.SECONDS));


(pulsar) 18/27: [fix][fn] Remove deprecated github.com/golang/protobuf dependency from pulsar-function-go (#26587)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit d0b8f53228b54d19722440988e73d4036ef38dbf
Author: Marius Gassen <ma...@users.noreply.github.com>
AuthorDate: Tue Sep 15 18:23:54 2026 +0200

    [fix][fn] Remove deprecated github.com/golang/protobuf dependency from pulsar-function-go (#26587)
    
    (cherry picked from commit 511fc9db7224de24669e19c6e0bf857102498ac1)
---
 pulsar-function-go/go.mod           | 1 -
 pulsar-function-go/pf/instance.go   | 8 +++-----
 pulsar-function-go/pf/stats_test.go | 6 +++---
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod
index 576577dea104..0c08f4129fb4 100644
--- a/pulsar-function-go/go.mod
+++ b/pulsar-function-go/go.mod
@@ -4,7 +4,6 @@ go 1.25.0
 
 require (
 	github.com/apache/pulsar-client-go v0.20.0
-	github.com/golang/protobuf v1.5.4
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/client_model v0.6.1
 	github.com/sirupsen/logrus v1.9.3
diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go
index 8624120d5780..f2b5dfd50b80 100644
--- a/pulsar-function-go/pf/instance.go
+++ b/pulsar-function-go/pf/instance.go
@@ -27,13 +27,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/golang/protobuf/ptypes/empty"
-
 	"github.com/apache/pulsar-client-go/pulsar"
-
 	log "github.com/apache/pulsar/pulsar-function-go/logutil"
 	pb "github.com/apache/pulsar/pulsar-function-go/pb"
 	prometheus_client "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/types/known/emptypb"
 )
 
 type goInstance struct {
@@ -648,9 +646,9 @@ func (gi *goInstance) getAndResetMetrics() *pb.MetricsData {
 	return metricsData
 }
 
-func (gi *goInstance) resetMetrics() *empty.Empty {
+func (gi *goInstance) resetMetrics() *emptypb.Empty {
 	gi.stats.reset()
-	return &empty.Empty{}
+	return &emptypb.Empty{}
 }
 
 // This method is used to get the required metrics for Prometheus.
diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go
index 138dc91cd9cd..550ef18aa8f0 100644
--- a/pulsar-function-go/pf/stats_test.go
+++ b/pulsar-function-go/pf/stats_test.go
@@ -28,10 +28,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/encoding/prototext"
+	"google.golang.org/protobuf/types/known/emptypb"
 
 	prometheus_client "github.com/prometheus/client_model/go"
 )
@@ -196,7 +196,7 @@ func TestInstanceControlMetrics(t *testing.T) {
 	instance := newGoInstance()
 	t.Cleanup(instance.close)
 	instanceClient := instanceCommunicationClient(t, instance)
-	_, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{})
+	_, err := instanceClient.GetMetrics(context.Background(), &emptypb.Empty{})
 	assert.NoError(t, err, "err communicating with instance control: %v", err)
 
 	testLabels := []string{"userMetricControlTest1", "userMetricControlTest2"}
@@ -209,7 +209,7 @@ func TestInstanceControlMetrics(t *testing.T) {
 	}
 	time.Sleep(time.Second)
 
-	metrics, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{})
+	metrics, err := instanceClient.GetMetrics(context.Background(), &emptypb.Empty{})
 	assert.NoError(t, err, "err communicating with instance control: %v", err)
 	for value, label := range testLabels {
 		assert.Containsf(t, metrics.UserMetrics, label, "user metrics should contain metric %s", label)


(pulsar) 22/27: [fix][broker] Release the compaction buffers and permits on flush failures (#26576)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 38384a36bc0f4642e6ad6aaf577e961605332088
Author: Zixuan Liu <no...@gmail.com>
AuthorDate: Sat Sep 19 06:17:59 2026 +0800

    [fix][broker] Release the compaction buffers and permits on flush failures (#26576)
    
    (cherry picked from commit 52ddf4a3bcb7452a283265dbbf2f398dfeadbcc3)
---
 .../client/impl/RawBatchMessageContainerImpl.java  | 12 ++++---
 .../compaction/StrategicTwoPhaseCompactor.java     | 25 +++++++++++++--
 .../impl/RawBatchMessageContainerImplTest.java     | 37 ++++++++++++++++++++++
 3 files changed, 67 insertions(+), 7 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
index 962aa4e1d2bc..be080b7faf3f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java
@@ -29,7 +29,6 @@ import org.apache.pulsar.client.api.MessageCrypto;
 import org.apache.pulsar.client.api.MessageIdAdv;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.impl.crypto.MessageCryptoBc;
-import org.apache.pulsar.common.allocator.PulsarByteBufAllocator;
 import org.apache.pulsar.common.api.EncryptionContext;
 import org.apache.pulsar.common.api.proto.CompressionType;
 import org.apache.pulsar.common.api.proto.MessageIdData;
@@ -196,6 +195,7 @@ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl {
 
         ByteBuf encryptedPayload = encrypt(getCompressedBatchMetadataAndPayload(false));
         ByteBuf metadataAndPayload = null;
+        ByteBuf buf = null;
         try {
             updateAndReserveBatchAllocatedSize(encryptedPayload.capacity());
             metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c,
@@ -211,15 +211,17 @@ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl {
             int idSize = idData.getSerializedSize();
             int headerSize = 4 /* IdSize */ + idSize + 4 /* metadataAndPayloadSize */;
             int totalSize = headerSize + metadataAndPayload.readableBytes();
-            ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(totalSize);
+            buf = allocator.buffer(totalSize);
             buf.writeInt(idSize);
             idData.writeTo(buf);
             buf.writeInt(metadataAndPayload.readableBytes());
             buf.writeBytes(metadataAndPayload);
-            return buf;
+            ByteBuf result = buf;
+            buf = null;
+            return result;
         } finally {
-            // Release everything allocated for this serialization on both success and failure, so a failure after
-            // the batch buffer was built (e.g. an OOM) cannot orphan it.
+            // buf is nulled on the success path, where its ownership moved to the returned buffer.
+            ReferenceCountUtil.safeRelease(buf);
             ReferenceCountUtil.safeRelease(metadataAndPayload);
             ReferenceCountUtil.safeRelease(encryptedPayload);
             clear();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
index c0cfd67ce4db..c051e48c422c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.compaction;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.util.ReferenceCountUtil;
 import java.time.Duration;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -440,23 +441,43 @@ public class StrategicTwoPhaseCompactor extends PublishingOrderCompactor {
             return CompletableFuture.completedFuture(false);
         }
         CompletableFuture<Boolean> bkf = new CompletableFuture<>();
+        ByteBuf serialized = null;
+        boolean acquired = false;
         try {
-            ByteBuf serialized = batchMessageContainer.toByteBuf();
+            serialized = batchMessageContainer.toByteBuf();
             outstanding.acquire();
+            acquired = true;
             mxBean.addCompactionWriteOp(topic, serialized.readableBytes());
             long start = System.nanoTime();
             lh.asyncAddEntry(serialized,
                     (rc, ledger, eid, ctx) -> {
                         outstanding.release();
-                        mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS);
+                        try {
+                            mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS);
+                        } catch (Throwable t) {
+                            // This callback can run synchronously inside asyncAddEntry: an exception escaping
+                            // it would reach the caller's catch, which would release the permit and the
+                            // buffer a second time - both already belong to the completed write here.
+                            log.warn("Failed to record the compaction write latency", t);
+                        }
                         if (rc != BKException.Code.OK) {
                             bkf.completeExceptionally(BKException.create(rc));
                         } else {
                             bkf.complete(true);
                         }
                     }, null);
+            // Ownership transferred to BookKeeper's write; it releases the buffer when done.
+            serialized = null;
 
         } catch (Throwable t) {
+            // toByteBuf() already cleared the container: nobody else holds the buffer if the hand-off
+            // never happened.
+            ReferenceCountUtil.safeRelease(serialized);
+            // Release the permit only when it was taken: a lost one stalls the next flush on acquire(),
+            // a surplus one raises the outstanding-writes limit.
+            if (acquired) {
+                outstanding.release();
+            }
             log.error("Failed to add entry", t);
             // discard(Exception) cannot take an Error: wrap non-Exception Throwables so the batch container is
             // always cleared here and stays reusable after a failed flush.
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
index 96a69ca61616..5c417444d586 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java
@@ -21,12 +21,14 @@ package org.apache.pulsar.client.impl;
 
 import static org.apache.pulsar.common.api.proto.CompressionType.NONE;
 import static org.apache.pulsar.common.api.proto.CompressionType.ZSTD;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anySet;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 import static org.testng.AssertJUnit.assertFalse;
 import static org.testng.AssertJUnit.assertTrue;
@@ -41,6 +43,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.pulsar.client.api.CryptoKeyReader;
 import org.apache.pulsar.client.api.EncryptionKeyInfo;
 import org.apache.pulsar.client.api.MessageCrypto;
@@ -223,6 +226,40 @@ public class RawBatchMessageContainerImplTest {
         buf.release();
     }
 
+    /**
+     * The output buffer of {@code toByteBuf()} is allocated before the header writes; a failure between the
+     * allocation and the return (here: a failing final write) must not orphan it - the finally-block releases
+     * it, since its ownership never moved to the caller.
+     */
+    @Test
+    public void testToByteBufReleasesTheOutputBufferWhenSerializationFails() {
+        List<ByteBuf> outputs = new ArrayList<>();
+        AtomicInteger allocations = new AtomicInteger();
+        ByteBufAllocator allocator = mock(ByteBufAllocator.class);
+        doAnswer(invocation -> {
+            ByteBuf real = Unpooled.buffer((int) invocation.getArgument(0));
+            // The second allocation through the instance allocator is toByteBuf()'s output buffer (the first
+            // is the batch buffer from add()).
+            if (allocations.incrementAndGet() == 2) {
+                // Delegates everything but fails the final writeBytes, standing in for a write that throws.
+                ByteBuf failing = spy(real);
+                doThrow(new RuntimeException("mocked write failure")).when(failing).writeBytes(any(ByteBuf.class));
+                outputs.add(failing);
+                return failing;
+            }
+            return real;
+        }).when(allocator).buffer(anyInt());
+
+        RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(allocator);
+        container.add(createMessage("my-topic", "hi", 0), null);
+
+        assertThatThrownBy(container::toByteBuf)
+                .hasMessageContaining("mocked write failure");
+        Assert.assertEquals(outputs.size(), 1);
+        Assert.assertEquals(outputs.get(0).refCnt(), 0,
+                "the output buffer must be released when the serialization fails after its allocation");
+    }
+
     @Test
     public void testToByteBufWithSingleMessage() throws IOException {
         RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl();


(pulsar) 03/27: [fix][client] Enable TableView compacted reads for short topic names (#26626)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit c2bee15de40cb7a750787f9dca9150a2f0029d08
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Thu Sep 17 13:51:37 2026 +0300

    [fix][client] Enable TableView compacted reads for short topic names (#26626)
    
    (cherry picked from commit 6c8fc014a7322e2325add9805a531d96ef1274ea)
---
 .../apache/pulsar/client/impl/TableViewImpl.java   |  3 +-
 .../client/impl/TableViewBuilderImplTest.java      |  2 +
 .../pulsar/client/impl/TableViewImplTest.java      | 48 ++++++++++++++++++++++
 3 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
index 998dbafd73c3..b9ebf3f3bac7 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java
@@ -44,6 +44,7 @@ import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TableView;
 import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.common.naming.TopicDomain;
+import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.topics.TopicCompactionStrategy;
 
 @Slf4j
@@ -83,7 +84,7 @@ public class TableViewImpl<T> implements TableView<T> {
 
     TableViewImpl(PulsarClientImpl client, Schema<T> schema, TableViewConfigurationData conf) {
         this.conf = conf;
-        this.isPersistentTopic = conf.getTopicName().startsWith(TopicDomain.persistent.toString());
+        this.isPersistentTopic = TopicName.get(conf.getTopicName()).getDomain() == TopicDomain.persistent;
         this.data = new ConcurrentHashMap<>();
         this.immutableData = Collections.unmodifiableMap(data);
         this.listeners = new ArrayList<>();
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java
index e2becec1105e..9a2120cec3b6 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertNotNull;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
@@ -52,6 +53,7 @@ public class TableViewBuilderImplTest {
         Reader reader = mock(Reader.class);
         readNextFuture = new CompletableFuture();
         when(reader.readNextAsync()).thenReturn(readNextFuture);
+        when(reader.getLastMessageIdsAsync()).thenReturn(CompletableFuture.completedFuture(List.of()));
         client = mock(PulsarClientImpl.class);
         ConnectionPool connectionPool = mock(ConnectionPool.class);
         when(client.getCnxPool()).thenReturn(connectionPool);
diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java
index 9761cc9aed31..eea34f2415ea 100644
--- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java
+++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java
@@ -23,6 +23,8 @@ import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
@@ -42,6 +44,7 @@ import org.apache.pulsar.client.api.Reader;
 import org.apache.pulsar.client.api.ReaderBuilder;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TableView;
+import org.apache.pulsar.client.api.TopicMessageId;
 import org.apache.pulsar.common.topics.TopicCompactionStrategy;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
@@ -71,6 +74,51 @@ public class TableViewImplTest {
 
         assertNotNull(tableView);
     }
+    @DataProvider
+    public Object[][] topicNames() {
+        return new Object[][]{
+                {"persistent://tenant/ns/topic", true},
+                {"tenant/ns/topic", true},
+                {"topic", true},
+                {"persistent-topic", true},
+                {"non-persistent://tenant/ns/topic", false}
+        };
+    }
+
+    @Test(timeOut = 10_000, dataProvider = "topicNames")
+    @SuppressWarnings("unchecked")
+    public void testTopicDomain(String topic, boolean persistent) throws Exception {
+        PulsarClientImpl client = mock(PulsarClientImpl.class);
+        ReaderBuilder<String> builder = mock(ReaderBuilder.class, RETURNS_SELF);
+        Reader<String> reader = mock(Reader.class);
+        when(client.newReader(Schema.STRING)).thenReturn(builder);
+        when(builder.createAsync()).thenReturn(CompletableFuture.completedFuture(reader));
+        when(reader.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
+        CompletableFuture<List<TopicMessageId>> lastMessageIds =
+                new CompletableFuture<>();
+        when(reader.getLastMessageIdsAsync()).thenReturn(lastMessageIds);
+        when(reader.readNextAsync()).thenReturn(new CompletableFuture<>());
+        TableViewConfigurationData conf = new TableViewConfigurationData();
+        conf.setTopicName(topic);
+        try (TableViewImpl<String> tableView = new TableViewImpl<>(client, Schema.STRING, conf)) {
+            if (persistent) {
+                verify(builder).readCompacted(true);
+            } else {
+                verify(builder, never()).readCompacted(true);
+            }
+            var start = tableView.start();
+            assertEquals(start.isDone(), !persistent,
+                    "Persistent topics must wait for the initial replay");
+            lastMessageIds.complete(List.of());
+            assertEquals(start.get(5, TimeUnit.SECONDS), tableView);
+            if (persistent) {
+                verify(reader).getLastMessageIdsAsync();
+            } else {
+                verify(reader, never()).getLastMessageIdsAsync();
+            }
+        }
+    }
+
     @DataProvider
     public Object[][] skippedMessage() {
         return new Object[][]{{false}, {true}};


(pulsar) 16/27: [fix][broker] Avoid load shedding and metadata writes from a former leader (#26253)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 1815b88ebb5bb04699bb2be02f04b5744e139b63
Author: void-ptr974 <vo...@protonmail.com>
AuthorDate: Sun Sep 13 03:29:04 2026 +0800

    [fix][broker] Avoid load shedding and metadata writes from a former leader (#26253)
    
    Co-authored-by: Lari Hotari <lh...@apache.org>
    (cherry picked from commit af2c40d46bcde518c3083e26c76eeada0a078052)
---
 .../loadbalance/impl/ModularLoadManagerImpl.java   | 165 ++++++++++++-----
 .../impl/ModularLoadManagerImplTest.java           | 206 ++++++++++++++++++++-
 2 files changed, 328 insertions(+), 43 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 411be8a33aa2..2b2afa8a95e3 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -38,7 +38,6 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -171,7 +170,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
 
     private Map<String, String> brokerToFailureDomainMap;
 
-    private SessionEvent lastMetadataSessionEvent = SessionEvent.Reconnected;
+    private volatile SessionEvent lastMetadataSessionEvent = SessionEvent.Reconnected;
 
     // record load balancing metrics
     private AtomicReference<List<Metrics>> loadBalancingMetrics = new AtomicReference<>();
@@ -299,8 +298,34 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
         }
     }
 
-    private void handleMetadataSessionEvent(SessionEvent e) {
+    @VisibleForTesting
+    void handleMetadataSessionEvent(SessionEvent e) {
+        boolean wasConnected = isMetadataSessionConnected();
         lastMetadataSessionEvent = e;
+        if (wasConnected && !e.isConnected()
+                && pulsar.getLeaderElectionService() != null
+                && pulsar.getLeaderElectionService().isLeader()) {
+            log.warn("Metadata session disconnected while this broker is the load manager leader;"
+                    + " leader-only operations will be skipped. metadataSessionEvent={}", e);
+        }
+    }
+
+    /**
+     * Best-effort guard for leader-only operations in the modular load manager. The metadata session listener and
+     * load-manager tasks run on different threads, so the session event must be visible here. Requiring a connected
+     * session prevents this manager from acting on a leader-election state that can remain stale while disconnected.
+     * This local check does not change the shared leader-election semantics or fence a leadership change that happens
+     * after the check.
+     */
+    @VisibleForTesting
+    boolean isLeader() {
+        return isMetadataSessionConnected()
+                && pulsar.getLeaderElectionService() != null
+                && pulsar.getLeaderElectionService().isLeader();
+    }
+
+    private boolean isMetadataSessionConnected() {
+        return lastMetadataSessionEvent != null && lastMetadataSessionEvent.isConnected();
     }
 
     private LoadSheddingStrategy createLoadSheddingStrategy() {
@@ -308,6 +333,16 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
                 Thread.currentThread().getContextClassLoader());
     }
 
+    @VisibleForTesting
+    void setLoadSheddingStrategy(LoadSheddingStrategy loadSheddingStrategy) {
+        this.loadSheddingStrategy = loadSheddingStrategy;
+    }
+
+    @VisibleForTesting
+    void setBundleSplitStrategy(BundleSplitStrategy bundleSplitStrategy) {
+        this.bundleSplitStrategy = bundleSplitStrategy;
+    }
+
     /**
      * Initialize this load manager.
      *
@@ -504,9 +539,16 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
         Collection<String> deadBrokers = CollectionUtils.subtract(knownBrokers, activeBrokers);
         this.knownBrokers.clear();
         this.knownBrokers.addAll(activeBrokers);
-        if (pulsar.getLeaderElectionService() != null
-                && pulsar.getLeaderElectionService().isLeader()) {
-            deadBrokers.forEach(this::deleteTimeAverageDataFromMetadataStoreAsync);
+        if (isLeader()) {
+            for (String deadBroker : deadBrokers) {
+                if (!isLeader()) {
+                    break;
+                }
+                deleteTimeAverageDataFromMetadataStoreAsync(deadBroker);
+            }
+            if (!isLeader()) {
+                return;
+            }
             loadSheddingStrategy.onActiveBrokersChange(activeBrokers);
             placementStrategy.onActiveBrokersChange(activeBrokers);
         }
@@ -606,7 +648,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
         for (String bundle : bundleData.keySet()) {
             if (!activeBundles.contains(bundle)){
                 bundleData.remove(bundle);
-                if (pulsar.getLeaderElectionService() != null && pulsar.getLeaderElectionService().isLeader()){
+                if (isLeader()) {
                     deleteBundleDataFromMetadataStore(bundle);
                 }
             }
@@ -640,6 +682,9 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
      */
     @Override
     public synchronized void doLoadShedding() {
+        if (!isLeader()) {
+            return;
+        }
         if (!LoadManagerShared.isLoadSheddingEnabled(pulsar)) {
             return;
         }
@@ -658,9 +703,15 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
             final Multimap<String, String> bundlesToUnload =
                     loadSheddingStrategy.findBundlesForUnloading(loadData, conf);
 
-            bundlesToUnload.asMap().forEach((broker, bundles) -> {
-                AtomicBoolean unloadBundleForBroker = new AtomicBoolean(false);
-                bundles.forEach(bundle -> {
+            boolean leadershipLost = false;
+            for (Map.Entry<String, Collection<String>> entry : bundlesToUnload.asMap().entrySet()) {
+                String broker = entry.getKey();
+                boolean unloadBundleForBroker = false;
+                for (String bundle : entry.getValue()) {
+                    if (!isLeader()) {
+                        leadershipLost = true;
+                        break;
+                    }
                     final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                     final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
                     if (sheddingExcludedNamespaces.contains(namespaceName)) {
@@ -668,44 +719,50 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
                             log.debug("[{}] Skipping load shedding for namespace {}",
                                     loadSheddingStrategy.getClass().getSimpleName(), namespaceName);
                         }
-                        return;
+                        continue;
                     }
                     if (!shouldNamespacePoliciesUnload(namespaceName, bundleRange, broker)) {
-                        return;
+                        continue;
                     }
 
                     if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) {
-                        return;
+                        continue;
                     }
                     NamespaceBundle bundleToUnload = LoadManagerShared.getNamespaceBundle(pulsar, bundle);
                     Optional<String> destBroker = this.selectBroker(bundleToUnload);
                     if (!destBroker.isPresent()) {
                         log.info("[{}] No broker available to unload bundle {} from broker {}",
                                 loadSheddingStrategy.getClass().getSimpleName(), bundle, broker);
-                        return;
+                        continue;
                     }
                     if (destBroker.get().equals(broker)) {
                         log.warn("[{}] The destination broker {} is the same as the current owner broker for Bundle {}",
                                 loadSheddingStrategy.getClass().getSimpleName(), destBroker.get(), bundle);
-                        return;
+                        continue;
                     }
 
-                    log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}",
-                            loadSheddingStrategy.getClass().getSimpleName(), bundle, broker, destBroker.get());
                     try {
-                        pulsar.getAdminClient().namespaces()
-                                .unloadNamespaceBundle(namespaceName, bundleRange, destBroker.get());
+                        if (!isLeader()) {
+                            leadershipLost = true;
+                            break;
+                        }
+                        log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}",
+                                loadSheddingStrategy.getClass().getSimpleName(), bundle, broker, destBroker.get());
+                        unloadNamespaceBundle(namespaceName, bundleRange, destBroker.get());
                         loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis());
                         unloadBundleCount++;
-                        unloadBundleForBroker.set(true);
+                        unloadBundleForBroker = true;
                     } catch (PulsarServerException | PulsarAdminException e) {
                         log.warn("Error when trying to perform load shedding on {} for broker {}", bundle, broker, e);
                     }
-                });
-                if (unloadBundleForBroker.get()) {
+                }
+                if (unloadBundleForBroker) {
                     unloadBrokerCount++;
                 }
-            });
+                if (leadershipLost) {
+                    break;
+                }
+            }
         } finally {
             loadSheddingStrategy.onUnloadAttemptCompleted();
         }
@@ -713,6 +770,12 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
         updateBundleUnloadingMetrics();
     }
 
+    @VisibleForTesting
+    void unloadNamespaceBundle(String namespaceName, String bundleRange, String destinationBroker)
+            throws PulsarServerException, PulsarAdminException {
+        pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange, destinationBroker);
+    }
+
     /**
      * As leader broker, update bundle unloading metrics.
      */
@@ -774,8 +837,7 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
     @Override
     public void checkNamespaceBundleSplit() {
 
-        if (!conf.isLoadBalancerAutoBundleSplitEnabled() || pulsar.getLeaderElectionService() == null
-                || !pulsar.getLeaderElectionService().isLeader() || knownBrokers.size() <= 1) {
+        if (!conf.isLoadBalancerAutoBundleSplitEnabled() || !isLeader() || knownBrokers.size() <= 1) {
             return;
         }
         final boolean unloadSplitBundles = pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
@@ -798,14 +860,6 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
                         continue;
                     }
 
-                    // Make sure the same bundle is not selected again.
-                    loadData.getBundleData().remove(bundleName);
-                    localData.getLastStats().remove(bundleName);
-                    // Clear namespace bundle-cache
-                    this.pulsar.getNamespaceService().getNamespaceBundleFactory()
-                            .invalidateBundleCache(NamespaceName.get(namespaceName));
-                    deleteBundleDataFromMetadataStore(bundleName);
-
                     // Check NamespacePolicies and AntiAffinityNamespace support unload bundle.
                     boolean isUnload = false;
                     String broker = bundlesToBeSplit.get(bundleName);
@@ -814,10 +868,21 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
                             && shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) {
                         isUnload = true;
                     }
+                    if (!isLeader()) {
+                        break;
+                    }
                     log.info("Load-manager splitting bundle {} and unloading {}", bundleName, isUnload);
                     pulsar.getAdminClient().namespaces().splitNamespaceBundle(namespaceName, bundleRange,
                             isUnload, null);
 
+                    // The split succeeded, so the old bundle must not be selected again.
+                    loadData.getBundleData().remove(bundleName);
+                    localData.getLastStats().remove(bundleName);
+                    // Clear namespace bundle-cache
+                    this.pulsar.getNamespaceService().getNamespaceBundleFactory()
+                            .invalidateBundleCache(NamespaceName.get(namespaceName));
+                    deleteBundleDataFromMetadataStore(bundleName);
+
                     splitCount++;
                     log.info("Successfully split namespace bundle {}", bundleName);
                 } catch (Exception e) {
@@ -1221,26 +1286,42 @@ public class ModularLoadManagerImpl implements ModularLoadManager {
      */
     @Override
     public void writeBundleDataOnZooKeeper() {
+        if (!isLeader()) {
+            return;
+        }
         updateBundleData();
+        if (!isLeader()) {
+            return;
+        }
         // Write the bundle data to metadata store.
         List<CompletableFuture<Void>> futures = new ArrayList<>();
+        boolean leadershipLost = false;
 
         // use synchronized to protect bundleArr.
         synchronized (bundleArr) {
             int updateBundleCount = selectTopKBundle();
-            bundleArr.stream().limit(updateBundleCount).forEach(entry -> futures.add(
-                    pulsarResources.getLoadBalanceResources().getBundleDataResources().updateBundleData(
-                            entry.getKey(), (BundleData) entry.getValue())));
+            for (Map.Entry<String, ? extends Comparable> entry : bundleArr.subList(0, updateBundleCount)) {
+                if (!isLeader()) {
+                    leadershipLost = true;
+                    break;
+                }
+                futures.add(pulsarResources.getLoadBalanceResources().getBundleDataResources().updateBundleData(
+                        entry.getKey(), (BundleData) entry.getValue()));
+            }
         }
 
         // Write the time average broker data to metadata store.
-        for (Map.Entry<String, BrokerData> entry : loadData.getBrokerData().entrySet()) {
-            final String broker = entry.getKey();
-            final TimeAverageBrokerData data = entry.getValue().getTimeAverageData();
-            futures.add(pulsarResources.getLoadBalanceResources()
-                    .getBrokerTimeAverageDataResources().updateTimeAverageBrokerData(broker, data));
+        if (!leadershipLost) {
+            for (Map.Entry<String, BrokerData> entry : loadData.getBrokerData().entrySet()) {
+                if (!isLeader()) {
+                    break;
+                }
+                final String broker = entry.getKey();
+                final TimeAverageBrokerData data = entry.getValue().getTimeAverageData();
+                futures.add(pulsarResources.getLoadBalanceResources()
+                        .getBrokerTimeAverageDataResources().updateTimeAverageBrokerData(broker, data));
+            }
         }
-
         try {
             FutureUtil.waitForAll(futures).join();
         } catch (Exception e) {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
index dfc0cc6219b3..03a41473eaa5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
@@ -59,14 +60,17 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.Level;
 import org.apache.pulsar.broker.PulsarServerException;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.BundleSplitStrategy;
 import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils;
 import org.apache.pulsar.broker.loadbalance.LoadData;
 import org.apache.pulsar.broker.loadbalance.LoadManager;
@@ -93,12 +97,14 @@ import org.apache.pulsar.common.policies.data.ClusterData;
 import org.apache.pulsar.common.policies.data.NamespaceIsolationDataImpl;
 import org.apache.pulsar.common.policies.data.ResourceQuota;
 import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.stats.Metrics;
 import org.apache.pulsar.common.util.ObjectMapperFactory;
 import org.apache.pulsar.common.util.PortManager;
 import org.apache.pulsar.metadata.api.MetadataCache;
 import org.apache.pulsar.metadata.api.Notification;
 import org.apache.pulsar.metadata.api.NotificationType;
 import org.apache.pulsar.metadata.api.extended.CreateOption;
+import org.apache.pulsar.metadata.api.extended.SessionEvent;
 import org.apache.pulsar.policies.data.loadbalancer.BrokerData;
 import org.apache.pulsar.policies.data.loadbalancer.BundleData;
 import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData;
@@ -108,6 +114,7 @@ import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
 import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData;
 import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData;
 import org.apache.pulsar.utils.ResourceUtils;
+import org.apache.pulsar.utils.TestLogAppender;
 import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
 import org.awaitility.Awaitility;
 import org.mockito.Mockito;
@@ -369,6 +376,202 @@ public class ModularLoadManagerImplTest {
         return String.format("%d/%d/0x00000000_0xffffffff", i, i);
     }
 
+    @Test
+    public void testFollowerSkipsLoadShedding() {
+        Awaitility.await().until(() -> pulsar1.getLeaderElectionService().isLeader()
+                || pulsar2.getLeaderElectionService().isLeader());
+
+        ModularLoadManagerImpl followerLoadManager = pulsar1.getLeaderElectionService().isLeader()
+                ? secondaryLoadManager : primaryLoadManager;
+
+        LoadSheddingStrategy loadSheddingStrategy = Mockito.mock(LoadSheddingStrategy.class);
+        followerLoadManager.setLoadSheddingStrategy(loadSheddingStrategy);
+
+        followerLoadManager.doLoadShedding();
+
+        verifyNoInteractions(loadSheddingStrategy);
+    }
+
+    @Test
+    public void testMetadataSessionDisconnectionInvalidatesLeadershipAndLogsOnce() throws Exception {
+        Awaitility.await().until(() -> pulsar1.getLeaderElectionService().isLeader()
+                || pulsar2.getLeaderElectionService().isLeader());
+
+        ModularLoadManagerImpl leaderLoadManager = spy(pulsar1.getLeaderElectionService().isLeader()
+                ? primaryLoadManager : secondaryLoadManager);
+        assertTrue(leaderLoadManager.isLeader());
+
+        try (TestLogAppender appender = TestLogAppender.create(ModularLoadManagerImpl.class)) {
+            leaderLoadManager.handleMetadataSessionEvent(SessionEvent.ConnectionLost);
+            leaderLoadManager.handleMetadataSessionEvent(SessionEvent.SessionLost);
+
+            assertFalse(leaderLoadManager.isLeader());
+            assertEquals(appender.getEvents().stream()
+                    .filter(event -> event.getLevel() == Level.WARN)
+                    .filter(event -> event.getMessage().getFormattedMessage()
+                            .contains("leader-only operations will be skipped"))
+                    .count(), 1L);
+        }
+    }
+
+    @Test
+    public void testLoadSheddingStopsWhenLeadershipChangesBeforeUnload() throws Exception {
+        Awaitility.await().until(() -> primaryLoadManager.getAvailableBrokers().size() > 1);
+
+        AtomicBoolean leader = new AtomicBoolean(true);
+        ModularLoadManagerImpl loadManagerSpy = spy(primaryLoadManager);
+        doAnswer(invocation -> leader.get()).when(loadManagerSpy).isLeader();
+
+        LoadSheddingStrategy loadSheddingStrategy = Mockito.mock(LoadSheddingStrategy.class);
+        loadManagerSpy.setLoadSheddingStrategy(loadSheddingStrategy);
+        when(loadSheddingStrategy.findBundlesForUnloading(any(), any()))
+                .thenReturn(ImmutableMultimap.of(primaryBrokerId, mockBundleName(1),
+                        primaryBrokerId, mockBundleName(2)));
+        doAnswer(invocation -> true).when(loadManagerSpy).shouldNamespacePoliciesUnload(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+        doAnswer(invocation -> true).when(loadManagerSpy).shouldAntiAffinityNamespaceUnload(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+        doAnswer(invocation -> {
+            leader.set(false);
+            return Optional.of(secondaryBrokerId);
+        }).when(loadManagerSpy).selectBroker(any());
+        doNothing().when(loadManagerSpy).unloadNamespaceBundle(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+
+        loadManagerSpy.doLoadShedding();
+
+        verify(loadSheddingStrategy).onUnloadAttemptCompleted();
+
+        verify(loadManagerSpy, Mockito.times(1)).selectBroker(any());
+        verify(loadManagerSpy, Mockito.never()).unloadNamespaceBundle(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+    }
+
+    @Test
+    public void testLoadSheddingPublishesMetricsWhenLeadershipChangesAfterUnload() throws Exception {
+        Awaitility.await().until(() -> primaryLoadManager.getAvailableBrokers().size() > 1);
+
+        String firstBundle = mockBundleName(1);
+        String secondBundle = mockBundleName(2);
+        AtomicBoolean leader = new AtomicBoolean(true);
+        ModularLoadManagerImpl loadManagerSpy = spy(primaryLoadManager);
+        doAnswer(invocation -> leader.get()).when(loadManagerSpy).isLeader();
+
+        LoadSheddingStrategy loadSheddingStrategy = Mockito.mock(LoadSheddingStrategy.class);
+        loadManagerSpy.setLoadSheddingStrategy(loadSheddingStrategy);
+        when(loadSheddingStrategy.findBundlesForUnloading(any(), any()))
+                .thenReturn(ImmutableMultimap.of(primaryBrokerId, firstBundle, primaryBrokerId, secondBundle));
+        doAnswer(invocation -> true).when(loadManagerSpy).shouldNamespacePoliciesUnload(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+        doAnswer(invocation -> true).when(loadManagerSpy).shouldAntiAffinityNamespaceUnload(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+        doAnswer(invocation -> Optional.of(secondaryBrokerId)).when(loadManagerSpy).selectBroker(any());
+        doAnswer(invocation -> {
+            leader.set(false);
+            return null;
+        }).when(loadManagerSpy).unloadNamespaceBundle(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+
+        loadManagerSpy.doLoadShedding();
+
+        verify(loadSheddingStrategy).onUnloadAttemptCompleted();
+
+        verify(loadManagerSpy, Mockito.times(1)).unloadNamespaceBundle(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+        assertTrue(loadManagerSpy.getLoadData().getRecentlyUnloadedBundles().containsKey(firstBundle));
+        assertFalse(loadManagerSpy.getLoadData().getRecentlyUnloadedBundles().containsKey(secondBundle));
+        Metrics unloadMetrics = loadManagerSpy.getLoadBalancingMetrics().stream()
+                .filter(metric -> "bundleUnloading".equals(metric.getDimension("metric")))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(unloadMetrics.getMetrics().get("brk_lb_unload_broker_total"), 1L);
+        assertEquals(unloadMetrics.getMetrics().get("brk_lb_unload_bundle_total"), 1L);
+    }
+
+    @Test
+    public void testBundleSplitDoesNotCleanUpWhenLeadershipIsLostBeforeSplit() throws Exception {
+        Awaitility.await().until(() -> primaryLoadManager.getAvailableBrokers().size() > 1);
+        pulsar1.getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
+        pulsar1.getConfiguration().setLoadBalancerAutoUnloadSplitBundlesEnabled(true);
+        primaryLoadManager.updateAll();
+
+        String tenant = "split-leadership-loss";
+        String namespace = "test";
+        admin1.clusters().createCluster("use", ClusterData.builder()
+                .serviceUrl(pulsar1.getWebServiceAddress()).build());
+        admin1.tenants().createTenant(tenant,
+                new TenantInfoImpl(Set.of("appid1"), Set.of("use")));
+        admin1.namespaces().createNamespace(tenant + "/" + namespace);
+
+        String topic = "persistent://" + tenant + "/" + namespace + "/topic";
+        String bundle = pulsar1.getNamespaceService().getBundle(TopicName.get(topic)).toString();
+        BundleData bundleData = new BundleData(10, 1000);
+        primaryLoadManager.getLoadData().getBundleData().put(bundle, bundleData);
+        String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundle);
+        MetadataCache<BundleData> metadataCache = pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class);
+        metadataCache.create(bundleDataPath, bundleData).join();
+
+        ModularLoadManagerImpl loadManagerSpy = spy(primaryLoadManager);
+        BundleSplitStrategy splitStrategy = (__, ___) -> Map.of(bundle, primaryBrokerId);
+        loadManagerSpy.setBundleSplitStrategy(splitStrategy);
+        AtomicBoolean leader = new AtomicBoolean(true);
+        doAnswer(invocation -> leader.get()).when(loadManagerSpy).isLeader();
+        doAnswer(invocation -> {
+            leader.set(false);
+            return true;
+        }).when(loadManagerSpy).shouldNamespacePoliciesUnload(
+                Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+
+        loadManagerSpy.checkNamespaceBundleSplit();
+
+        assertTrue(loadManagerSpy.getLoadData().getBundleData().containsKey(bundle));
+        assertTrue(metadataCache.get(bundleDataPath).join().isPresent());
+    }
+
+    @Test
+    public void testBundleDataWriteStopsWhenLeadershipChangesBeforeMetadataWrite() throws Exception {
+        String bundle = mockBundleName(99);
+        BundleData bundleData = new BundleData(10, 1000);
+        String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundle);
+        MetadataCache<BundleData> metadataCache = pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class);
+        metadataCache.create(bundleDataPath, bundleData).join();
+
+        Awaitility.await().until(() -> primaryLoadManager.getLoadData().getBrokerData().containsKey(primaryBrokerId));
+        AtomicInteger leaderChecks = new AtomicInteger();
+        ModularLoadManagerImpl loadManagerSpy = spy(primaryLoadManager);
+        LoadData loadData = loadManagerSpy.getLoadData();
+        loadData.getBundleData().clear();
+        loadData.getBundleData().put(bundle, bundleData);
+        loadData.getBrokerData().get(primaryBrokerId).getLocalData().getLastStats()
+                .put(bundle, new NamespaceBundleStats());
+        doAnswer(invocation -> leaderChecks.getAndIncrement() < 2).when(loadManagerSpy).isLeader();
+
+        loadManagerSpy.writeBundleDataOnZooKeeper();
+
+        assertEquals(metadataCache.getWithStats(bundleDataPath).get().get().getStat().getVersion(), 0);
+    }
+
+    @Test
+    public void testBundleDataAggregationDoesNotDeleteAfterLeadershipLoss() throws Exception {
+        String bundle = mockBundleName(100);
+        BundleData bundleData = new BundleData(10, 1000);
+        String bundleDataPath = String.format("%s/%s", BUNDLE_DATA_BASE_PATH, bundle);
+        MetadataCache<BundleData> metadataCache = pulsar1.getLocalMetadataStore().getMetadataCache(BundleData.class);
+        metadataCache.create(bundleDataPath, bundleData).join();
+
+        Awaitility.await().until(() -> primaryLoadManager.getLoadData().getBrokerData().containsKey(primaryBrokerId));
+        AtomicInteger leaderChecks = new AtomicInteger();
+        ModularLoadManagerImpl loadManagerSpy = spy(primaryLoadManager);
+        LoadData loadData = loadManagerSpy.getLoadData();
+        loadData.getBundleData().clear();
+        loadData.getBundleData().put(bundle, bundleData);
+        doAnswer(invocation -> leaderChecks.getAndIncrement() == 0).when(loadManagerSpy).isLeader();
+
+        loadManagerSpy.writeBundleDataOnZooKeeper();
+
+        assertTrue(metadataCache.get(bundleDataPath).join().isPresent());
+    }
+
     // Test disabled since it's depending on CPU usage in the machine
     @Test(enabled = false)
     public void testCandidateConsistency() throws Exception {
@@ -690,7 +893,8 @@ public class ModularLoadManagerImplTest {
         pulsar3.start();
 
         ModularLoadManagerWrapper loadManagerWrapper = (ModularLoadManagerWrapper) pulsar3.getLoadManager().get();
-        ModularLoadManagerImpl loadManager = (ModularLoadManagerImpl) loadManagerWrapper.getLoadManager();
+        ModularLoadManagerImpl loadManager = spy((ModularLoadManagerImpl) loadManagerWrapper.getLoadManager());
+        doAnswer(invocation -> true).when(loadManager).isLeader();
         Awaitility.await().untilAsserted(() -> assertTrue(loadManager.getAvailableBrokers().size() > 1));
 
         NamespaceBundle bundle = makeBundle("test", "load-shedding-strategy");


(pulsar) 07/27: [fix][ml] Fix backlog estimation during ledger rollover with pending writes (#26647)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 8d785b7acbf7d9b7bf913f1103e10be7375d5417
Author: sinan liu <li...@gmail.com>
AuthorDate: Sun Sep 20 10:20:44 2026 +0800

    [fix][ml] Fix backlog estimation during ledger rollover with pending writes (#26647)
    
    Co-authored-by: Lari Hotari <lh...@users.noreply.github.com>
    (cherry picked from commit 776248a7fe0f93b213b9f06658ca7f7e64c87c42)
---
 .../bookkeeper/mledger/impl/ManagedCursorImpl.java |  14 ++-
 .../mledger/impl/ManagedLedgerBkTest.java          | 109 +++++++++++++++++++++
 2 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 501487366d29..85357e5067d0 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -1312,7 +1312,7 @@ public class ManagedCursorImpl implements ManagedCursor {
         }
         if (markDeletePosition.compareTo(lastPosition) > 0) {
             if (!ledger.ledgerExists(lastPosition.getLedgerId())
-                    || isMarkDeletePositionOnEmptyCurrentLedger(markDeletePosition)) {
+                    || isMarkDeletePositionAtStartOfCurrentLedger(markDeletePosition)) {
                 return 0;
             }
             throw new IllegalArgumentException(String.format(
@@ -1385,10 +1385,14 @@ public class ManagedCursorImpl implements ManagedCursor {
         return adjustedSize;
     }
 
-    private boolean isMarkDeletePositionOnEmptyCurrentLedger(Position markDeletePosition) {
-        return ledger.currentLedger != null
-                && markDeletePosition.getLedgerId() == ledger.currentLedger.getId()
-                && ledger.currentLedgerEntries == 0;
+    private boolean isMarkDeletePositionAtStartOfCurrentLedger(Position markDeletePosition) {
+        // The caller has already checked that mark-delete is ahead of the last confirmed position.
+        // Rollover can move the cursor to this sentinel while the first add is still pending;
+        // currentLedgerEntries includes pending adds, so it cannot identify this state.
+        LedgerHandle currentLedger = ledger.currentLedger;
+        return currentLedger != null
+                && markDeletePosition.getLedgerId() == currentLedger.getId()
+                && markDeletePosition.getEntryId() == -1;
     }
 
     private long getNumberOfEntriesInBacklog() {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
index 9740db647d90..0d086b6be66c 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
@@ -20,6 +20,7 @@ package org.apache.bookkeeper.mledger.impl;
 
 import static org.apache.pulsar.common.util.PortManager.releaseLockedPort;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertNotNull;
@@ -79,6 +80,114 @@ public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
         super(2);
     }
 
+    private static ManagedLedgerConfig defaultConfig() {
+        ManagedLedgerConfig config = new ManagedLedgerConfig();
+        config.setPulsarMessageEntries(false);
+        return config;
+    }
+
+    @Test
+    public void testEstimatedUnackedSizeDuringRolloverWithPendingAdd() throws Exception {
+        @Cleanup("shutdown")
+        ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+        ManagedLedgerConfig config = defaultConfig()
+                .setEnsembleSize(2).setWriteQuorumSize(2).setAckQuorumSize(2)
+                .setRetentionSizeInMB(-1).setRetentionTime(-1, TimeUnit.MILLISECONDS);
+        config.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("pending-add-" + testName, config);
+        ManagedCursor cursor = ledger.openCursor("c1");
+        byte[] data = "entry".getBytes(StandardCharsets.UTF_8);
+        ledger.addEntry(data);
+        Position lastPosition = ledger.addEntry(data);
+        cursor.markDelete(lastPosition);
+        config.setMaxEntriesPerLedger(2);
+
+        CompletableFuture<Position> pendingAdd = new CompletableFuture<>();
+        CountDownLatch resumeBookie = new CountDownLatch(1);
+        CountDownLatch bookieSuspended = new CountDownLatch(1);
+        try {
+            // Both bookies must acknowledge the write. Hold one so the first add to the new ledger stays pending.
+            sleepBookie(getBookie(0), resumeBookie, bookieSuspended);
+            assertThat(bookieSuspended.await(10, TimeUnit.SECONDS)).isTrue();
+            ledger.rollCurrentLedgerIfFull();
+            ledger.asyncAddEntry(data, new AddEntryCallback() {
+                @Override
+                public void addComplete(Position position, ByteBuf entryData, Object ctx) {
+                    pendingAdd.complete(position);
+                }
+
+                @Override
+                public void addFailed(ManagedLedgerException exception, Object ctx) {
+                    pendingAdd.completeExceptionally(exception);
+                }
+            }, null);
+
+            // Rollover itself moves the caught-up cursor to the new ledger's -1 sentinel.
+            Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+                assertThat(cursor.getMarkDeletedPosition().getLedgerId()).isGreaterThan(lastPosition.getLedgerId());
+                assertThat(cursor.getMarkDeletedPosition().getEntryId()).isEqualTo(-1);
+                assertThat(ledger.getCurrentLedgerEntries()).isEqualTo(1);
+            });
+            assertThat(cursor.getMarkDeletedPosition().getLedgerId()).isEqualTo(ledger.currentLedger.getId());
+            assertThat(ledger.getLastPosition()).isEqualTo(lastPosition);
+            // Keep the old ledger so the removed-ledger shortcut cannot mask the regression.
+            assertThat(ledger.ledgerExists(lastPosition.getLedgerId())).isTrue();
+            assertThat(pendingAdd).isNotDone();
+            assertThat(cursor.getEstimatedSizeSinceMarkDeletePosition()).isZero();
+        } finally {
+            resumeBookie.countDown();
+        }
+
+        Position newPosition = pendingAdd.get(10, TimeUnit.SECONDS);
+        assertThat(newPosition.getLedgerId()).isGreaterThan(lastPosition.getLedgerId());
+        assertThat(newPosition.getEntryId()).isZero();
+        assertThat(cursor.getEstimatedSizeSinceMarkDeletePosition()).isEqualTo(data.length);
+        List<Entry> entries = cursor.readEntries(1);
+        try {
+            assertThat(entries).hasSize(1);
+            assertThat(entries.get(0).getPosition()).isEqualTo(newPosition);
+            assertThat(entries.get(0).getData()).isEqualTo(data);
+        } finally {
+            entries.forEach(Entry::release);
+        }
+        cursor.markDelete(newPosition);
+        assertThat(cursor.getEstimatedSizeSinceMarkDeletePosition()).isZero();
+    }
+
+    @Test
+    public void testEstimatedUnackedSizeRejectsUnconfirmedEntryInEmptyLedger() throws Exception {
+        @Cleanup("shutdown")
+        ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
+        ManagedLedgerConfig config = defaultConfig()
+                .setEnsembleSize(2).setWriteQuorumSize(2).setAckQuorumSize(2)
+                .setRetentionSizeInMB(-1).setRetentionTime(-1, TimeUnit.MILLISECONDS);
+        config.setMinimumRolloverTime(0, TimeUnit.MILLISECONDS);
+        ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("invalid-position-" + testName, config);
+        ManagedCursor cursor = ledger.openCursor("c1");
+        Position lastPosition = ledger.addEntry("entry".getBytes(StandardCharsets.UTF_8));
+        cursor.markDelete(lastPosition);
+        config.setMaxEntriesPerLedger(1);
+        ledger.rollCurrentLedgerIfFull();
+
+        Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertThat(cursor.getMarkDeletedPosition().getLedgerId()).isGreaterThan(lastPosition.getLedgerId());
+            assertThat(cursor.getMarkDeletedPosition().getEntryId()).isEqualTo(-1);
+        });
+        assertThat(cursor.getEstimatedSizeSinceMarkDeletePosition()).isZero();
+
+        // asyncMarkDelete currently accepts an ahead-of-LAC entry in the next ledger after consuming the old one.
+        // The estimator must not mistake that entry for the valid -1 sentinel just because the ledger is empty.
+        Position invalidPosition = PositionFactory.create(cursor.getMarkDeletedPosition().getLedgerId(), 0);
+        cursor.markDelete(invalidPosition);
+        assertThat(cursor.getMarkDeletedPosition()).isEqualTo(invalidPosition);
+        assertThat(ledger.getCurrentLedgerEntries()).isZero();
+        assertThat(ledger.getLastPosition()).isEqualTo(lastPosition);
+        assertThat(ledger.ledgerExists(lastPosition.getLedgerId())).isTrue();
+        assertThatThrownBy(cursor::getEstimatedSizeSinceMarkDeletePosition)
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("is ahead of the last position");
+    }
+
     @Test
     public void testSimpleRead() throws Exception {
         ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();


(pulsar) 02/27: [fix][client] Flush acknowledgment groups at the combined size limit (#26594)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 3335340e346655d010537c3806cdbb730e49f6e9
Author: sinan liu <li...@gmail.com>
AuthorDate: Wed Sep 16 22:21:27 2026 +0800

    [fix][client] Flush acknowledgment groups at the combined size limit (#26594)
    
    (cherry picked from commit 33436e027957cf1fed79ba214e78d508adfaac71)
---
 .../api/ConsumerAcknowledgmentGroupSizeTest.java   | 126 +++++++++++++++++++++
 .../PersistentAcknowledgmentsGroupingTracker.java  |  11 +-
 2 files changed, 134 insertions(+), 3 deletions(-)

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAcknowledgmentGroupSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAcknowledgmentGroupSizeTest.java
new file mode 100644
index 000000000000..5ae3d7b241cd
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ConsumerAcknowledgmentGroupSizeTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.broker.service.SharedPulsarBaseTest;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-api")
+public class ConsumerAcknowledgmentGroupSizeTest extends SharedPulsarBaseTest {
+
+    @DataProvider
+    public Object[][] acknowledgmentGroups() {
+        return new Object[][] {
+            {true, true, true},
+            {true, false, true},
+            {true, true, false},
+            {false, false, true},
+            {false, true, false}
+        };
+    }
+
+    @Test(dataProvider = "acknowledgmentGroups", timeOut = 60000)
+    public void testFlushAtMaxGroupSize(boolean listAck, boolean firstBatch, boolean secondBatch) throws Exception {
+        String topic = newTopicName();
+        try (Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
+                     .topic(topic)
+                     .subscriptionName("sub")
+                     .subscriptionType(SubscriptionType.Shared)
+                     .enableBatchIndexAcknowledgment(true)
+                     .isAckReceiptEnabled(true)
+                     .maxAcknowledgmentGroupSize(2)
+                     .acknowledgmentGroupTime(1, TimeUnit.HOURS)
+                     .subscribe();
+             Producer<String> batchProducer = pulsarClient.newProducer(Schema.STRING)
+                     .topic(topic)
+                     .enableBatching(true)
+                     .batchingMaxMessages(2)
+                     .batchingMaxPublishDelay(1, TimeUnit.HOURS)
+                     .create();
+             Producer<String> individualProducer = pulsarClient.newProducer(Schema.STRING)
+                     .topic(topic)
+                     .enableBatching(false)
+                     .create()) {
+            List<MessageId> toAcknowledge = new ArrayList<>();
+            Set<String> unacknowledged = new HashSet<>();
+            boolean[] batches = {firstBatch, secondBatch};
+            for (int entry = 0; entry < batches.length; entry++) {
+                String value = "entry-" + entry;
+                if (batches[entry]) {
+                    CompletableFuture<MessageId> first = batchProducer.sendAsync(value + "-0");
+                    CompletableFuture<MessageId> second = batchProducer.sendAsync(value + "-1");
+                    batchProducer.flush();
+                    CompletableFuture.allOf(first, second).get(10, TimeUnit.SECONDS);
+                    unacknowledged.add(value + "-1");
+                } else {
+                    individualProducer.send(value + "-0");
+                }
+                Message<String> first = consumer.receive(10, TimeUnit.SECONDS);
+                assertThat(first).isNotNull();
+                assertThat(first.getValue()).isEqualTo(value + "-0");
+                MessageIdAdv firstId = (MessageIdAdv) first.getMessageId();
+                assertThat(firstId.getBatchIndex()).isEqualTo(batches[entry] ? 0 : -1);
+                toAcknowledge.add(firstId);
+                if (batches[entry]) {
+                    Message<String> second = consumer.receive(10, TimeUnit.SECONDS);
+                    assertThat(second).isNotNull();
+                    assertThat(second.getValue()).isEqualTo(value + "-1");
+                    MessageIdAdv secondId = (MessageIdAdv) second.getMessageId();
+                    assertThat(secondId.getLedgerId()).isEqualTo(firstId.getLedgerId());
+                    assertThat(secondId.getEntryId()).isEqualTo(firstId.getEntryId());
+                    assertThat(secondId.getBatchIndex()).isEqualTo(1);
+                    assertThat(firstId.getBatchSize()).isEqualTo(2);
+                }
+            }
+            MessageIdAdv firstId = (MessageIdAdv) toAcknowledge.get(0);
+            MessageIdAdv secondId = (MessageIdAdv) toAcknowledge.get(1);
+            assertThat(firstId.getEntryId()).isNotEqualTo(secondId.getEntryId());
+
+            CompletableFuture<Void> acknowledgment;
+            if (listAck) {
+                acknowledgment = consumer.acknowledgeAsync(toAcknowledge);
+            } else {
+                CompletableFuture<Void> first = consumer.acknowledgeAsync(toAcknowledge.get(0));
+                assertThat(first.isDone()).as("one pending entry stays below the group size limit").isFalse();
+                acknowledgment = CompletableFuture.allOf(first, consumer.acknowledgeAsync(toAcknowledge.get(1)));
+            }
+            // This completes only after the broker returns an ACK receipt. The one-hour timer cannot help.
+            acknowledgment.get(10, TimeUnit.SECONDS);
+
+            consumer.redeliverUnacknowledgedMessages();
+            int remaining = unacknowledged.size();
+            for (int i = 0; i < remaining; i++) {
+                Message<String> redelivered = consumer.receive(10, TimeUnit.SECONDS);
+                assertThat(redelivered).isNotNull();
+                assertThat(unacknowledged.remove(redelivered.getValue()))
+                        .as("only the unacknowledged batch indexes are redelivered").isTrue();
+            }
+            assertThat(unacknowledged).isEmpty();
+            assertThat(consumer.receive(500, TimeUnit.MILLISECONDS)).isNull();
+        }
+    }
+}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
index 0a366a759d3f..047e1e78e8f2 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java
@@ -166,7 +166,7 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
                 }
             } finally {
                 readLock.ifPresent(Lock::unlock);
-                if (acknowledgementGroupTimeMicros == 0 || pendingIndividualAcks.size() >= maxAckGroupSize) {
+                if (acknowledgementGroupTimeMicros == 0 || isMaxAckGroupSizeReached()) {
                     flush();
                 }
             }
@@ -266,7 +266,7 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
                 return readLock.map(__ -> currentIndividualAckFuture).orElse(CompletableFuture.completedFuture(null));
             } finally {
                 readLock.ifPresent(Lock::unlock);
-                if (pendingIndividualAcks.size() >= maxAckGroupSize) {
+                if (isMaxAckGroupSizeReached()) {
                     flush();
                 }
             }
@@ -274,6 +274,11 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
     }
 
 
+    private boolean isMaxAckGroupSizeReached() {
+        // Both collections are drained by the same flush; count pending entries across the entire group.
+        return (long) pendingIndividualAcks.size() + pendingIndividualBatchIndexAcks.size() >= maxAckGroupSize;
+    }
+
     private CompletableFuture<Void> doIndividualAckAsync(MessageIdAdv messageId) {
         pendingIndividualAcks.add(messageId);
         pendingIndividualBatchIndexAcks.remove(messageId);
@@ -302,7 +307,7 @@ public class PersistentAcknowledgmentsGroupingTracker implements Acknowledgments
             return readLock.map(__ -> currentIndividualAckFuture).orElse(CompletableFuture.completedFuture(null));
         } finally {
             readLock.ifPresent(Lock::unlock);
-            if (pendingIndividualBatchIndexAcks.size() >= maxAckGroupSize) {
+            if (isMaxAckGroupSizeReached()) {
                 flush();
             }
         }


(pulsar) 14/27: [fix][fn] Return descriptive errors for unknown JSON properties (#26441)

Posted by lh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 0d92b74c5e08ef3391c7dfaf7299c96fae76b1e4
Author: zgzyq <13...@users.noreply.github.com>
AuthorDate: Sat Sep 12 04:38:15 2026 +0800

    [fix][fn] Return descriptive errors for unknown JSON properties (#26441)
    
    (cherry picked from commit 8174b236a4df128a9d6169954a8d12d00ddf82de)
---
 .../pulsar/functions/worker/rest/Resources.java    |  3 ++
 .../functions/worker/rest/ResourcesTest.java       | 32 ++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/Resources.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/Resources.java
index d98ae68d9666..4c8dedbd28ae 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/Resources.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/Resources.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.functions.worker.rest;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import org.apache.pulsar.broker.web.UnrecognizedPropertyExceptionMapper;
 import org.apache.pulsar.functions.worker.rest.api.FunctionsMetricsResource;
 import org.apache.pulsar.functions.worker.rest.api.v2.FunctionsApiV2Resource;
 import org.apache.pulsar.functions.worker.rest.api.v2.WorkerApiV2Resource;
@@ -43,6 +44,7 @@ public final class Resources {
                         FunctionsApiV2Resource.class,
                         WorkerApiV2Resource.class,
                         WorkerStatsApiV2Resource.class,
+                        UnrecognizedPropertyExceptionMapper.class,
                         MultiPartFeature.class
                 ));
     }
@@ -51,6 +53,7 @@ public final class Resources {
         return new HashSet<>(
                 Arrays.asList(
                         MultiPartFeature.class,
+                        UnrecognizedPropertyExceptionMapper.class,
                         SourcesApiV3Resource.class,
                         SourceApiV3Resource.class,
                         SinksApiV3Resource.class,
diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/ResourcesTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/ResourcesTest.java
new file mode 100644
index 000000000000..e7ff36068438
--- /dev/null
+++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/ResourcesTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.worker.rest;
+
+import static org.testng.Assert.assertTrue;
+import org.apache.pulsar.broker.web.UnrecognizedPropertyExceptionMapper;
+import org.testng.annotations.Test;
+
+public class ResourcesTest {
+
+    @Test
+    public void testJsonExceptionMapperIsRegisteredForApiResources() {
+        assertTrue(Resources.getApiV2Resources().contains(UnrecognizedPropertyExceptionMapper.class));
+        assertTrue(Resources.getApiV3Resources().contains(UnrecognizedPropertyExceptionMapper.class));
+    }
+}