You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/06/09 07:01:02 UTC

(pulsar) branch master updated: [improve][broker] Default managedLedgerMaxReadsInFlightSizeInMB to 15% of direct memory (#25979)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 083fd4cb6fa [improve][broker] Default managedLedgerMaxReadsInFlightSizeInMB to 15% of direct memory (#25979)
083fd4cb6fa is described below

commit 083fd4cb6fa7585268a0534ecbc70746348e316b
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Jun 9 10:00:56 2026 +0300

    [improve][broker] Default managedLedgerMaxReadsInFlightSizeInMB to 15% of direct memory (#25979)
---
 conf/broker.conf                                   | 19 ++++++----
 conf/standalone.conf                               | 13 +++++++
 .../apache/pulsar/broker/ServiceConfiguration.java | 15 +++++---
 .../pulsar/broker/ManagedLedgerClientFactory.java  | 41 +++++++++++++++-------
 .../api/KeySharedSubscriptionBrokerCacheTest.java  |  2 +-
 ...ySharedSubscriptionDisabledBrokerCacheTest.java |  2 +-
 6 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 8ed01cc8656..4826bbeb5c7 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1473,13 +1473,18 @@ managedLedgerCursorMaxEntriesPerLedger=50000
 # Max time before triggering a rollover on a cursor ledger
 managedLedgerCursorRolloverTimeInSeconds=14400
 
-# Maximum amount of memory used hold data read from storage (or from the cache).
-# This mechanism prevents the broker to have too many concurrent
-# reads from storage and fall into Out of Memory errors in case
-# of multiple concurrent reads to multiple concurrent consumers.
-# Set 0 in order to disable the feature.
-#
-managedLedgerMaxReadsInFlightSizeInMB=0
+# Maximum buffer size in MB for bytes read from storage (or from the cache).
+# This is the memory retained by data read from storage (or cache) until it has been
+# delivered to the Consumer Netty channel. This provides backpressure for BookKeeper and
+# tiered storage reads, preventing the broker from having too many concurrent reads and
+# running into Out of Memory errors when there are multiple concurrent reads to multiple
+# concurrent consumers.
+# When left unset (empty), it defaults to the greater of dispatcherMaxReadSizeBytes and 15% of available
+# JVM direct memory; dispatcherMaxReadSizeBytes is the minimum value so the limiter can never block the
+# completion of a single read.
+# Set to 0 to disable the feature.
+# Set to a value greater than 0 to use that many MB.
+managedLedgerMaxReadsInFlightSizeInMB=
 
 # Max number of "acknowledgment holes" that are going to be persistently stored.
 # When acknowledging out of order, a consumer will leave holes that are supposed
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 38c5d1a5b89..2dba365a5a7 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -914,6 +914,19 @@ managedLedgerCacheEvictionExtendTTLOfRecentlyAccessed=true
 # The default value is 2 * managedLedgerCacheEvictionTimeThresholdMillis.
 managedLedgerContinueCachingAddedEntriesAfterLastActiveCursorLeavesMillis=
 
+# Maximum buffer size in MB for bytes read from storage (or from the cache).
+# This is the memory retained by data read from storage (or cache) until it has been
+# delivered to the Consumer Netty channel. This provides backpressure for BookKeeper and
+# tiered storage reads, preventing the broker from having too many concurrent reads and
+# running into Out of Memory errors when there are multiple concurrent reads to multiple
+# concurrent consumers.
+# When left unset (empty), it defaults to the greater of dispatcherMaxReadSizeBytes and 15% of available
+# JVM direct memory; dispatcherMaxReadSizeBytes is the minimum value so the limiter can never block the
+# completion of a single read.
+# Set to 0 to disable the feature.
+# Set to a value greater than 0 to use that many MB.
+managedLedgerMaxReadsInFlightSizeInMB=
+
 # Configure the threshold (in number of entries) from where a cursor should be considered 'backlogged'
 # and thus should be set as inactive.
 # This has no effect when cacheEvictionByExpectedReadCount is enabled.
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 7771b43059f..48be38ad809 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -2335,10 +2335,17 @@ public class ServiceConfiguration implements PulsarConfiguration {
             + "inserting in cache")
     private boolean managedLedgerCacheCopyEntries = false;
 
-    @FieldContext(category = CATEGORY_STORAGE_ML, doc = "Maximum buffer size for bytes read from storage."
-            + " This is the memory retained by data read from storage (or cache) until it has been delivered to the"
-            + " Consumer Netty channel. Use O to disable")
-    private long managedLedgerMaxReadsInFlightSizeInMB = 0;
+    @FieldContext(category = CATEGORY_STORAGE_ML, doc = "Maximum buffer size in MB for bytes read from storage"
+            + " (or from the cache). This is the memory retained by data read from storage (or cache) until it has"
+            + " been delivered to the Consumer Netty channel. This provides backpressure for BookKeeper and tiered"
+            + " storage reads, preventing the broker from having too many concurrent reads and running into Out of"
+            + " Memory errors when there are multiple concurrent reads to multiple concurrent consumers.\n"
+            + "When left unset (empty), it defaults to the greater of dispatcherMaxReadSizeBytes and 15% of"
+            + " available JVM direct memory; dispatcherMaxReadSizeBytes is the minimum value so the limiter"
+            + " can never block the completion of a single read.\n"
+            + "Set to 0 to disable the feature.\n"
+            + "Set to a value greater than 0 to use that many MB.")
+    private Long managedLedgerMaxReadsInFlightSizeInMB = null;
 
     @FieldContext(category = CATEGORY_STORAGE_ML, doc = "Maximum time to wait for acquiring permits for max reads in "
             + "flight when managedLedgerMaxReadsInFlightSizeInMB is set (>0) and the limit is reached.")
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
index 0d168b4ca57..3dece00e89a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/ManagedLedgerClientFactory.java
@@ -48,6 +48,7 @@ import org.apache.pulsar.broker.storage.ManagedLedgerStorage;
 import org.apache.pulsar.broker.storage.ManagedLedgerStorageClass;
 import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig;
 import org.apache.pulsar.common.stats.CacheMetricsCollector;
+import org.apache.pulsar.common.util.DirectMemoryUtils;
 import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
 
 @CustomLog
@@ -90,18 +91,34 @@ public class ManagedLedgerClientFactory implements ManagedLedgerStorage {
             );
         }
         managedLedgerFactoryConfig.setCopyEntriesInCache(conf.isManagedLedgerCacheCopyEntries());
-        long managedLedgerMaxReadsInFlightSizeBytes = conf.getManagedLedgerMaxReadsInFlightSizeInMB() * 1024L * 1024L;
-        if (managedLedgerMaxReadsInFlightSizeBytes > 0 && conf.getDispatcherMaxReadSizeBytes() > 0
-                && managedLedgerMaxReadsInFlightSizeBytes < conf.getDispatcherMaxReadSizeBytes()) {
-            log.warn()
-                    .attr("managedLedgerMaxReadsInFlightSizeInMB", conf.getManagedLedgerMaxReadsInFlightSizeInMB())
-                    .attr("dispatcherMaxReadSizeBytes", conf.getDispatcherMaxReadSizeBytes())
-                    .attr("minManagedLedgerMaxReadsInFlightSizeInMB",
-                            (conf.getDispatcherMaxReadSizeBytes() / (1024L * 1024L)) + 1)
-                    .log("Invalid configuration:"
-                            + " managedLedgerMaxReadsInFlightSizeInMB in bytes"
-                            + " should be greater than"
-                            + " dispatcherMaxReadSizeBytes");
+        Long managedLedgerMaxReadsInFlightSizeInMB = conf.getManagedLedgerMaxReadsInFlightSizeInMB();
+        // A single dispatcher read can retain up to dispatcherMaxReadSizeBytes bytes. The in-flight reads
+        // limit must be at least this size, otherwise the limiter can block the completion of a single read.
+        long dispatcherMaxReadSizeBytes = conf.getDispatcherMaxReadSizeBytes();
+        final long managedLedgerMaxReadsInFlightSizeBytes;
+        if (managedLedgerMaxReadsInFlightSizeInMB == null) {
+            // When unset, default to 15% of the available JVM direct memory, but never below the maximum
+            // size of a single read (dispatcherMaxReadSizeBytes) so that the limiter can never block the
+            // completion of one read.
+            long fractionOfDirectMemory = (long) (0.15d * DirectMemoryUtils.jvmMaxDirectMemory());
+            managedLedgerMaxReadsInFlightSizeBytes = Math.max(fractionOfDirectMemory, dispatcherMaxReadSizeBytes);
+        } else {
+            // An explicit 0 disables the feature; an explicit value > 0 is used as-is.
+            managedLedgerMaxReadsInFlightSizeBytes = managedLedgerMaxReadsInFlightSizeInMB * 1024L * 1024L;
+            // Warn when the feature is enabled but manually configured below the size of a single read, since
+            // the limiter would then be unable to admit one read. Disabled (0) is ignored.
+            if (managedLedgerMaxReadsInFlightSizeBytes > 0 && dispatcherMaxReadSizeBytes > 0
+                    && managedLedgerMaxReadsInFlightSizeBytes < dispatcherMaxReadSizeBytes) {
+                log.warn()
+                        .attr("managedLedgerMaxReadsInFlightSizeInMB", managedLedgerMaxReadsInFlightSizeInMB)
+                        .attr("dispatcherMaxReadSizeBytes", dispatcherMaxReadSizeBytes)
+                        .attr("minManagedLedgerMaxReadsInFlightSizeInMB",
+                                (dispatcherMaxReadSizeBytes + (1024L * 1024L) - 1) / (1024L * 1024L))
+                        .log("Invalid configuration:"
+                                + " managedLedgerMaxReadsInFlightSizeInMB in bytes should be greater than or"
+                                + " equal to dispatcherMaxReadSizeBytes, otherwise the in-flight reads limiter"
+                                + " can block the completion of a single read.");
+            }
         }
         managedLedgerFactoryConfig.setManagedLedgerMaxReadsInFlightSize(managedLedgerMaxReadsInFlightSizeBytes);
         managedLedgerFactoryConfig.setManagedLedgerMaxReadsInFlightPermitsAcquireTimeoutMillis(
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionBrokerCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionBrokerCacheTest.java
index d78c78883b7..2419a86c7db 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionBrokerCacheTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionBrokerCacheTest.java
@@ -100,7 +100,7 @@ public class KeySharedSubscriptionBrokerCacheTest extends ProducerConsumerBase {
         // Important: this is currently necessary to make use of cache for replay queue reads
         conf.setCacheEvictionByMarkDeletedPosition(true);
 
-        conf.setManagedLedgerMaxReadsInFlightSizeInMB(100);
+        conf.setManagedLedgerMaxReadsInFlightSizeInMB(100L);
         conf.setDispatcherRetryBackoffInitialTimeInMs(0);
         conf.setDispatcherRetryBackoffMaxTimeInMs(0);
         conf.setKeySharedUnblockingIntervalMs(0);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionDisabledBrokerCacheTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionDisabledBrokerCacheTest.java
index 7a5b16df32a..f4160bc8a7a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionDisabledBrokerCacheTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionDisabledBrokerCacheTest.java
@@ -94,7 +94,7 @@ public class KeySharedSubscriptionDisabledBrokerCacheTest extends ProducerConsum
         this.conf.setUnblockStuckSubscriptionEnabled(false);
         this.conf.setSubscriptionKeySharedUseConsistentHashing(true);
         conf.setManagedLedgerCacheSizeMB(0);
-        conf.setManagedLedgerMaxReadsInFlightSizeInMB(0);
+        conf.setManagedLedgerMaxReadsInFlightSizeInMB(0L);
         conf.setDispatcherRetryBackoffInitialTimeInMs(0);
         conf.setDispatcherRetryBackoffMaxTimeInMs(0);
         conf.setKeySharedUnblockingIntervalMs(0);