You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2022/04/15 11:50:21 UTC
[pulsar] branch master updated: [cleanup][broker] PIP 45: Deprecate zookeeper settings managedLedgerMaxUnackedRangesToPersistInZooKeeper (#15099)
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8c534dbcb8d [cleanup][broker] PIP 45: Deprecate zookeeper settings managedLedgerMaxUnackedRangesToPersistInZooKeeper (#15099)
8c534dbcb8d is described below
commit 8c534dbcb8daab8a249501604a9312ff4b5a1166
Author: Qiang Zhao <74...@users.noreply.github.com>
AuthorDate: Fri Apr 15 19:50:13 2022 +0800
[cleanup][broker] PIP 45: Deprecate zookeeper settings managedLedgerMaxUnackedRangesToPersistInZooKeeper (#15099)
---
conf/broker.conf | 12 +++++++++---
conf/standalone.conf | 12 +++++++++---
deployment/terraform-ansible/templates/broker.conf | 12 +++++++++---
.../apache/bookkeeper/mledger/ManagedLedgerConfig.java | 10 +++++-----
.../bookkeeper/mledger/impl/ManagedCursorImpl.java | 2 +-
.../bookkeeper/mledger/impl/ManagedCursorTest.java | 4 ++--
.../org/apache/pulsar/broker/ServiceConfiguration.java | 16 +++++++++++++++-
.../org/apache/pulsar/broker/service/BrokerService.java | 4 ++--
8 files changed, 52 insertions(+), 20 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index e128632fcf7..8c503590869 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1025,10 +1025,10 @@ managedLedgerCursorRolloverTimeInSeconds=14400
# crashes.
managedLedgerMaxUnackedRangesToPersist=10000
-# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher
# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
-# zookeeper.
-managedLedgerMaxUnackedRangesToPersistInZooKeeper=1000
+# MetadataStore.
+managedLedgerMaxUnackedRangesToPersistInMetadataStore=1000
# Skip reading non-recoverable/unreadable data-ledger under managed-ledger's list. It helps when data-ledgers gets
# corrupted at bookkeeper and managed-cursor is stuck at that ledger.
@@ -1446,3 +1446,9 @@ tlsEnabled=false
# Enable Key_Shared subscription (default is enabled)
# @deprecated since 2.8.0 subscriptionTypesEnabled is preferred over subscriptionKeySharedEnable.
subscriptionKeySharedEnable=true
+
+# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
+# zookeeper.
+# Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore
+managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 3ea8664f695..d9dc896cab2 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -710,10 +710,10 @@ managedLedgerMaxSizePerLedgerMbytes=2048
# crashes.
managedLedgerMaxUnackedRangesToPersist=10000
-# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher
# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
-# zookeeper.
-managedLedgerMaxUnackedRangesToPersistInZooKeeper=1000
+# MetadataStore.
+managedLedgerMaxUnackedRangesToPersistInMetadataStore=1000
# Skip reading non-recoverable/unreadable data-ledger under managed-ledger's list. It helps when data-ledgers gets
# corrupted at bookkeeper and managed-cursor is stuck at that ledger.
@@ -1085,3 +1085,9 @@ zookeeperServers=
# Configuration Store connection string
configurationStoreServers=
+
+# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
+# zookeeper.
+# Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore
+managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
\ No newline at end of file
diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
index 3e1000cc4b1..fb4456f0b1d 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -825,10 +825,10 @@ managedLedgerCursorRolloverTimeInSeconds=14400
# crashes.
managedLedgerMaxUnackedRangesToPersist=10000
-# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher
# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
-# zookeeper.
-managedLedgerMaxUnackedRangesToPersistInZooKeeper=1000
+# MetadataStore.
+managedLedgerMaxUnackedRangesToPersistInMetadataStore=1000
# Skip reading non-recoverable/unreadable data-ledger under managed-ledger's list. It helps when data-ledgers gets
# corrupted at bookkeeper and managed-cursor is stuck at that ledger.
@@ -1131,6 +1131,12 @@ replicationTlsEnabled=false
# Deprecated. Use brokerDeleteInactiveTopicsFrequencySeconds
brokerServicePurgeInactiveFrequencyInSeconds=60
+# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher
+# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
+# zookeeper.
+# Deprecated: use managedLedgerMaxUnackedRangesToPersistInMetadataStore
+managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
+
### --- Transaction config variables --- ###
# Enable transaction coordinator in broker
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 8b0375d23f3..7da5f87ced5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -45,7 +45,7 @@ public class ManagedLedgerConfig {
private int maxUnackedRangesToPersist = 10000;
private int maxBatchDeletedIndexToPersist = 10000;
private boolean deletionAtBatchIndexLevelEnabled = true;
- private int maxUnackedRangesToPersistInZk = 1000;
+ private int maxUnackedRangesToPersistInMetadataStore = 1000;
private int maxEntriesPerLedger = 50000;
private int maxSizePerLedgerMb = 100;
private int minimumRolloverTimeMs = 0;
@@ -483,12 +483,12 @@ public class ManagedLedgerConfig {
* @return max unacked message ranges up to which it can store in Zookeeper
*
*/
- public int getMaxUnackedRangesToPersistInZk() {
- return maxUnackedRangesToPersistInZk;
+ public int getMaxUnackedRangesToPersistInMetadataStore() {
+ return maxUnackedRangesToPersistInMetadataStore;
}
- public void setMaxUnackedRangesToPersistInZk(int maxUnackedRangesToPersistInZk) {
- this.maxUnackedRangesToPersistInZk = maxUnackedRangesToPersistInZk;
+ public void setMaxUnackedRangesToPersistInMetadataStore(int maxUnackedRangesToPersistInMetadataStore) {
+ this.maxUnackedRangesToPersistInMetadataStore = maxUnackedRangesToPersistInMetadataStore;
}
/**
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 084c31641ec..9ac58f11508 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -2375,7 +2375,7 @@ public class ManagedCursorImpl implements ManagedCursor {
return cursorLedger != null
&& !isCursorLedgerReadOnly
&& config.getMaxUnackedRangesToPersist() > 0
- && individualDeletedMessages.size() > config.getMaxUnackedRangesToPersistInZk();
+ && individualDeletedMessages.size() > config.getMaxUnackedRangesToPersistInMetadataStore();
}
private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map<String, Long> properties,
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 6202a0f2a4e..382b2d066ac 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -2847,7 +2847,7 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
String cursorName = "c1";
ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
// metaStore is allowed to store only up to 10 deleted entries range
- managedLedgerConfig.setMaxUnackedRangesToPersistInZk(10);
+ managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(10);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
@@ -3237,7 +3237,7 @@ public class ManagedCursorTest extends MockedBookKeeperTestCase {
public void testBatchIndexesDeletionPersistAndRecover() throws ManagedLedgerException, InterruptedException {
ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
// Make sure the cursor metadata updated by the cursor ledger ID.
- managedLedgerConfig.setMaxUnackedRangesToPersistInZk(-1);
+ managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(-1);
ManagedLedger ledger = factory.open("test_batch_indexes_deletion_persistent", managedLedgerConfig);
ManagedCursor cursor = ledger.openCursor("c1");
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index f28e086a9e9..6851ee7c6ba 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1832,12 +1832,20 @@ public class ServiceConfiguration implements PulsarConfiguration {
+ " will only be tracked in memory and messages will be redelivered in case of"
+ " crashes.")
private int managedLedgerMaxUnackedRangesToPersist = 10000;
+ @Deprecated
@FieldContext(
category = CATEGORY_STORAGE_ML,
+ deprecated = true,
doc = "Max number of `acknowledgment holes` that can be stored in Zookeeper.\n\n"
+ "If number of unack message range is higher than this limit then broker will persist"
+ " unacked ranges into bookkeeper to avoid additional data overhead into zookeeper.")
- private int managedLedgerMaxUnackedRangesToPersistInZooKeeper = 1000;
+ private int managedLedgerMaxUnackedRangesToPersistInZooKeeper = -1;
+ @FieldContext(
+ category = CATEGORY_STORAGE_ML,
+ doc = "Max number of `acknowledgment holes` that can be stored in MetadataStore.\n\n"
+ + "If number of unack message range is higher than this limit then broker will persist"
+ + " unacked ranges into bookkeeper to avoid additional data overhead into MetadataStore.")
+ private int managedLedgerMaxUnackedRangesToPersistInMetadataStore = 1000;
@FieldContext(
category = CATEGORY_STORAGE_OFFLOADING,
doc = "Use Open Range-Set to cache unacked messages (it is memory efficient but it can take more cpu)"
@@ -2749,6 +2757,12 @@ public class ServiceConfiguration implements PulsarConfiguration {
return schemaCompatibilityStrategy;
}
+ public int getManagedLedgerMaxUnackedRangesToPersistInMetadataStore() {
+ return managedLedgerMaxUnackedRangesToPersistInZooKeeper > 0
+ ? managedLedgerMaxUnackedRangesToPersistInZooKeeper :
+ managedLedgerMaxUnackedRangesToPersistInMetadataStore;
+ }
+
public long getMetadataStoreSessionTimeoutMillis() {
return zooKeeperSessionTimeoutMillis > 0 ? zooKeeperSessionTimeoutMillis : metadataStoreSessionTimeoutMillis;
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 353496ecec3..27784332fdc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -1565,8 +1565,8 @@ public class BrokerService implements Closeable {
managedLedgerConfig
.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
- managedLedgerConfig.setMaxUnackedRangesToPersistInZk(
- serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInZooKeeper());
+ managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore(
+ serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore());
managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger());
managedLedgerConfig
.setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(),