You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by zi...@apache.org on 2024/10/29 03:55:49 UTC
(pulsar) 08/08: [feat][broker] Add ResourceGroup rate limit for topic dispatch and replication (#26)
This is an automated email from the ASF dual-hosted git repository.
zixuan pushed a commit to branch branch-as-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit a9672262041006b9674392b1c1784343a9c0ee7e
Author: Zixuan Liu <no...@gmail.com>
AuthorDate: Tue Oct 29 11:47:20 2024 +0800
[feat][broker] Add ResourceGroup rate limit for topic dispatch and replication (#26)
* [feat][broker] Add ResourceGroup rate limit for topic dispatch and replication
Signed-off-by: Zixuan Liu <no...@gmail.com>
* [fix][broker] Fix ownership loss
Signed-off-by: Zixuan Liu <no...@gmail.com>
* Fix test
Signed-off-by: Zixuan Liu <no...@gmail.com>
---------
Signed-off-by: Zixuan Liu <no...@gmail.com>
---
.../apache/pulsar/broker/ServiceConfiguration.java | 27 ++
.../broker/resources/ResourceGroupResources.java | 4 +
.../broker/admin/impl/PersistentTopicsBase.java | 36 +++
.../broker/admin/impl/ResourceGroupsBase.java | 11 +-
.../pulsar/broker/admin/v2/PersistentTopics.java | 56 ++++
.../pulsar/broker/namespace/OwnershipCache.java | 12 +-
.../pulsar/broker/resourcegroup/ResourceGroup.java | 175 ++++++++---
.../ResourceGroupDispatchLimiter.java | 172 +++++++++++
.../ResourceGroupRateLimiterManager.java | 67 +++++
.../broker/resourcegroup/ResourceGroupService.java | 335 ++++++++++++++++-----
.../resourcegroup/ResourceQuotaCalculatorImpl.java | 90 ++++--
.../pulsar/broker/service/AbstractTopic.java | 51 +++-
.../apache/pulsar/broker/service/Replicator.java | 5 +
.../org/apache/pulsar/broker/service/Topic.java | 5 +
.../service/nonpersistent/NonPersistentTopic.java | 6 +-
.../persistent/GeoPersistentReplicator.java | 28 +-
.../PersistentDispatcherMultipleConsumers.java | 22 ++
.../PersistentDispatcherSingleActiveConsumer.java | 25 ++
.../service/persistent/PersistentReplicator.java | 191 +++++++++---
.../broker/service/persistent/PersistentTopic.java | 16 +-
.../service/persistent/ShadowReplicator.java | 28 +-
pulsar-broker/src/main/proto/ResourceUsage.proto | 1 +
.../broker/admin/AdminResourceGroupTest.java | 168 +++++++++++
.../resourcegroup/ResourceGroupMetricTest.java | 49 +++
.../ResourceGroupRateLimiterManagerTest.java | 109 +++++++
.../resourcegroup/ResourceGroupServiceTest.java | 213 ++++++++++++-
...ourceGroupUsageAggregationOnTopicLevelTest.java | 228 ++++++++++++++
.../ResourceGroupUsageAggregationTest.java | 2 -
.../ResourceQuotaCalculatorImplTest.java | 13 +-
.../ResourceUsageTransportManagerTest.java | 8 +-
.../broker/service/ReplicatorRateLimiterTest.java | 152 ++++++++++
.../apache/pulsar/client/admin/ResourceGroups.java | 4 +
.../apache/pulsar/client/admin/TopicPolicies.java | 59 ++++
.../common/policies/data/ReplicatorStats.java | 10 +
.../pulsar/common/policies/data/ResourceGroup.java | 3 +
.../client/admin/internal/TopicPoliciesImpl.java | 49 +++
.../apache/pulsar/admin/cli/CmdResourceGroups.java | 20 ++
.../apache/pulsar/admin/cli/CmdTopicPolicies.java | 59 ++++
.../policies/data/HierarchyTopicPolicies.java | 3 +
.../pulsar/common/policies/data/TopicPolicies.java | 2 +
.../policies/data/stats/ReplicatorStatsImpl.java | 10 +
41 files changed, 2307 insertions(+), 217 deletions(-)
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ff401fdb2b7..4cbc3bfc409 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1109,6 +1109,33 @@ public class ServiceConfiguration implements PulsarConfiguration {
)
private int subscriptionPatternMaxLength = 50;
+ @FieldContext(
+ dynamic = true,
+ category = CATEGORY_POLICIES,
+ doc = "The percentage difference that is considered \"within limits\" to suppress usage reporting"
+ + "Setting this to 0 will also make us report in every round."
+ )
+ private int resourceUsageReportSuppressionTolerancePercentage = 5;
+
+ @FieldContext(
+ category = CATEGORY_POLICIES,
+ doc = "The maximum number of successive rounds that we can suppress reporting local usage, because there "
+ + "was no substantial change from the prior round. This is to ensure the reporting does not "
+ + "become too chatty. Set this value to one more than the cadence of sending reports; e.g., if "
+ + "you want to send every 3rd report, set the value to 4."
+ + "Setting this to 0 will make us report in every round."
+ + "Don't set to negative values; behavior will be disabled"
+ )
+ private int resourceUsageMaxUsageReportSuppressRounds = 5;
+
+ @FieldContext(
+ dynamic = true,
+ category = CATEGORY_POLICIES,
+ doc = "ResourceGroup rate limit will be triggered when the total traffic exceeds the product of the "
+ + "rate-limit value and the threshold. Value range: [0,1]."
+ )
+ private double resourceGroupLocalQuotaThreshold = 0;
+
// <-- dispatcher read settings -->
@FieldContext(
dynamic = true,
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
index 578609b2195..562261ff7ac 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
@@ -46,6 +46,10 @@ public class ResourceGroupResources extends BaseResources<ResourceGroup> {
return exists(joinPath(BASE_PATH, resourceGroupName));
}
+ public CompletableFuture<Boolean> resourceGroupExistsAsync(String resourceGroupName) {
+ return existsAsync(joinPath(BASE_PATH, resourceGroupName));
+ }
+
public void createResourceGroup(String resourceGroupName, ResourceGroup rg) throws MetadataStoreException {
create(joinPath(BASE_PATH, resourceGroupName), rg);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index abd9a37aaf7..65ef2a10a0e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -5682,4 +5682,40 @@ public class PersistentTopicsBase extends AdminResource {
return null;
}));
}
+
+ protected CompletableFuture<Void> internalSetResourceGroup(String resourceGroupName, boolean isGlobal) {
+ boolean isDelete = StringUtils.isEmpty(resourceGroupName);
+ return validateTopicPolicyOperationAsync(topicName, PolicyName.RESOURCEGROUP, PolicyOperation.WRITE)
+ .thenCompose(__ -> {
+ if (isDelete) {
+ return CompletableFuture.completedFuture(true);
+ }
+ return resourceGroupResources().resourceGroupExistsAsync(resourceGroupName);
+ })
+ .thenCompose(exists -> {
+ if (!exists) {
+ return FutureUtil.failedFuture(new RestException(Status.NOT_FOUND,
+ "ResourceGroup does not exist"));
+ }
+ return getTopicPoliciesAsyncWithRetry(topicName, isGlobal).thenCompose(op -> {
+ TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+ topicPolicies.setResourceGroupName(isDelete ? null : resourceGroupName);
+ topicPolicies.setIsGlobal(isGlobal);
+ return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+ });
+ });
+ }
+
+ protected CompletableFuture<String> internalGetResourceGroup(boolean applied, boolean isGlobal) {
+ return validateTopicPolicyOperationAsync(topicName, PolicyName.RESOURCEGROUP, PolicyOperation.READ)
+ .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)
+ .thenApply(op -> op.map(TopicPolicies::getResourceGroupName)
+ .orElseGet(() -> {
+ if (applied) {
+ return getNamespacePolicies(namespaceName).resource_group_name;
+ }
+ return null;
+ })
+ ));
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
index 826b9c322e3..9f984a35e11 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
@@ -75,7 +75,12 @@ public abstract class ResourceGroupsBase extends AdminResource {
if (rgConfig.getDispatchRateInBytes() != null) {
resourceGroup.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes());
}
-
+ if (rgConfig.getReplicationDispatchRateInBytes() != null) {
+ resourceGroup.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes());
+ }
+ if (rgConfig.getReplicationDispatchRateInMsgs() != null) {
+ resourceGroup.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs());
+ }
// write back the new ResourceGroup config.
resourceGroupResources().updateResourceGroup(rgName, r -> resourceGroup);
log.info("[{}] Successfully updated the ResourceGroup {}", clientAppId(), rgName);
@@ -96,6 +101,10 @@ public abstract class ResourceGroupsBase extends AdminResource {
? -1 : rgConfig.getDispatchRateInMsgs());
rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes() == null
? -1 : rgConfig.getDispatchRateInBytes());
+ rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes() == null
+ ? -1 : rgConfig.getReplicationDispatchRateInBytes());
+ rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs() == null
+ ? -1 : rgConfig.getReplicationDispatchRateInMsgs());
try {
resourceGroupResources().createResourceGroup(rgName, rgConfig);
log.info("[{}] Created ResourceGroup {}", clientAppId(), rgName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 2e1b943a2cc..bee27423228 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -4888,5 +4888,61 @@ public class PersistentTopics extends PersistentTopicsBase {
});
}
+ @POST
+ @Path("/{tenant}/{namespace}/{topic}/resourceGroup")
+ @ApiOperation(value = "Set ResourceGroup for a topic")
+ @ApiResponses(value = {
+ @ApiResponse(code = 403, message = "Don't have admin permission"),
+ @ApiResponse(code = 404, message = "Topic doesn't exist"),
+ @ApiResponse(code = 405, message =
+ "Topic level policy is disabled, enable the topic level policy and retry"),
+ @ApiResponse(code = 409, message = "Concurrent modification")
+ })
+ public void setResourceGroup(
+ @Suspended final AsyncResponse asyncResponse,
+ @PathParam("tenant") String tenant,
+ @PathParam("namespace") String namespace,
+ @PathParam("topic") String encodedTopic,
+ @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal,
+ @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
+ @ApiParam(value = "ResourceGroup name", required = true) String resourceGroupName) {
+ validateTopicName(tenant, namespace, encodedTopic);
+ preValidation(authoritative)
+ .thenCompose(__ -> internalSetResourceGroup(resourceGroupName, isGlobal))
+ .thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
+ .exceptionally(ex -> {
+ handleTopicPolicyException("setResourceGroup", ex, asyncResponse);
+ return null;
+ });
+ }
+
+ @GET
+ @Path("/{tenant}/{namespace}/{topic}/resourceGroup")
+ @ApiOperation(value = "Get ResourceGroup for a topic")
+ @ApiResponses(value = {
+ @ApiResponse(code = 403, message = "Don't have admin permission"),
+ @ApiResponse(code = 404, message = "Topic doesn't exist"),
+ @ApiResponse(code = 405, message =
+ "Topic level policy is disabled, enable the topic level policy and retry"),
+ @ApiResponse(code = 409, message = "Concurrent modification")
+ })
+ public void getResourceGroup(
+ @Suspended final AsyncResponse asyncResponse,
+ @PathParam("tenant") String tenant,
+ @PathParam("namespace") String namespace,
+ @PathParam("topic") String encodedTopic,
+ @QueryParam("applied") @DefaultValue("false") boolean applied,
+ @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal,
+ @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
+ validateTopicName(tenant, namespace, encodedTopic);
+ preValidation(authoritative)
+ .thenCompose(__ -> internalGetResourceGroup(applied, isGlobal))
+ .thenApply(asyncResponse::resume)
+ .exceptionally(ex -> {
+ handleTopicPolicyException("getResourceGroup", ex, asyncResponse);
+ return null;
+ });
+ }
+
private static final Logger log = LoggerFactory.getLogger(PersistentTopics.class);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
index 0033abf36c7..553fc95e9dc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
@@ -172,7 +172,17 @@ public class OwnershipCache {
// If we're not the owner, we need to check if anybody else is
String path = ServiceUnitUtils.path(suName);
- return lockManager.readLock(path);
+ return lockManager.readLock(path).thenCompose(owner -> {
+ //If the current broker is the owner, we must try to acquire ownership to avoid cache loss.
+ if (owner.isPresent() && owner.get().equals(selfOwnerInfo)) {
+ try {
+ return tryAcquiringOwnership(suName).thenApply(Optional::ofNullable);
+ } catch (Exception e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
+ return CompletableFuture.completedFuture(owner);
+ });
}
/**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
index 96a3a9d44b9..9a7e20a92e4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
@@ -18,15 +18,20 @@
*/
package org.apache.pulsar.broker.resourcegroup;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import io.prometheus.client.Counter;
-import java.util.HashMap;
+import io.prometheus.client.Gauge;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import lombok.Getter;
+import lombok.ToString;
import lombok.val;
+import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupOpStatus;
import org.apache.pulsar.broker.service.resource.usage.NetworkUsage;
import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
@@ -49,6 +54,7 @@ public class ResourceGroup {
/**
* Convenience class for bytes and messages counts, which are used together in a lot of the following code.
*/
+ @ToString
public static class BytesAndMessagesCount {
public long bytes;
public long messages;
@@ -61,6 +67,7 @@ public class ResourceGroup {
public enum ResourceGroupMonitoringClass {
Publish,
Dispatch,
+ ReplicationDispatch,
// Storage; // Punt this for now, until we have a clearer idea of the usage, statistics, etc.
}
@@ -70,7 +77,7 @@ public class ResourceGroup {
public enum ResourceGroupRefTypes {
Tenants,
Namespaces,
- // Topics; // Punt this for when we support direct ref/under from topics.
+ Topics
}
// Default ctor: it is not expected that anything outside of this package will need to directly
@@ -85,6 +92,13 @@ public class ResourceGroup {
this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor());
log.info("attaching publish rate limiter {} to {} get {}", this.resourceGroupPublishLimiter, name,
this.getResourceGroupPublishLimiter());
+ this.resourceGroupReplicationDispatchLimiter = ResourceGroupRateLimiterManager
+ .newReplicationDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+ log.info("attaching replication dispatch rate limiter {} to {}", this.resourceGroupReplicationDispatchLimiter,
+ name);
+ this.resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager
+ .newDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+ log.info("attaching topic dispatch rate limiter {} to {}", this.resourceGroupDispatchLimiter, name);
}
// ctor for overriding the transport-manager fill/set buffer.
@@ -98,6 +112,10 @@ public class ResourceGroup {
this.setResourceGroupMonitoringClassFields();
this.setResourceGroupConfigParameters(rgConfig);
this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor());
+ this.resourceGroupReplicationDispatchLimiter = ResourceGroupRateLimiterManager
+ .newReplicationDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+ this.resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager
+ .newDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
this.ruPublisher = rgPublisher;
this.ruConsumer = rgConsumer;
}
@@ -108,12 +126,15 @@ public class ResourceGroup {
this.resourceGroupName = other.resourceGroupName;
this.rgs = other.rgs;
this.resourceGroupPublishLimiter = other.resourceGroupPublishLimiter;
+ this.resourceGroupReplicationDispatchLimiter = other.resourceGroupReplicationDispatchLimiter;
+ this.resourceGroupDispatchLimiter = other.resourceGroupDispatchLimiter;
this.setResourceGroupMonitoringClassFields();
// ToDo: copy the monitoring class fields, and ruPublisher/ruConsumer from other, if required.
this.resourceGroupNamespaceRefs = other.resourceGroupNamespaceRefs;
this.resourceGroupTenantRefs = other.resourceGroupTenantRefs;
+ this.resourceGroupTopicRefs = other.resourceGroupTopicRefs;
for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) {
PerMonitoringClassFields thisFields = this.monitoringClassFields[idx];
@@ -146,12 +167,19 @@ public class ResourceGroup {
pubBmc.messages = rgConfig.getPublishRateInMsgs();
pubBmc.bytes = rgConfig.getPublishRateInBytes();
this.resourceGroupPublishLimiter.update(pubBmc);
+ ResourceGroupRateLimiterManager
+ .updateReplicationDispatchRateLimiter(resourceGroupReplicationDispatchLimiter, rgConfig);
+ ResourceGroupRateLimiterManager.updateDispatchRateLimiter(resourceGroupDispatchLimiter, rgConfig);
}
protected long getResourceGroupNumOfNSRefs() {
return this.resourceGroupNamespaceRefs.size();
}
+ protected long getResourceGroupNumOfTopicRefs() {
+ return this.resourceGroupTopicRefs.size();
+ }
+
protected long getResourceGroupNumOfTenantRefs() {
return this.resourceGroupTenantRefs.size();
}
@@ -169,6 +197,9 @@ public class ResourceGroup {
case Namespaces:
set = this.resourceGroupNamespaceRefs;
break;
+ case Topics:
+ set = this.resourceGroupTopicRefs;
+ break;
}
if (ref) {
@@ -179,7 +210,8 @@ public class ResourceGroup {
set.add(name);
// If this is the first ref, register with the transport manager.
- if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 1) {
+ if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size()
+ + this.resourceGroupTopicRefs.size() == 1) {
if (log.isDebugEnabled()) {
log.debug("registerUsage for RG={}: registering with transport-mgr", this.resourceGroupName);
}
@@ -194,7 +226,8 @@ public class ResourceGroup {
set.remove(name);
// If this was the last ref, unregister from the transport manager.
- if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 0) {
+ if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size()
+ + this.resourceGroupTopicRefs.size() == 0) {
if (log.isDebugEnabled()) {
log.debug("unRegisterUsage for RG={}: un-registering from transport-mgr", this.resourceGroupName);
}
@@ -226,6 +259,11 @@ public class ResourceGroup {
resourceUsage.clearDispatch();
}
+ p = resourceUsage.setReplicationDispatch();
+ if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, p)) {
+ resourceUsage.clearReplicationDispatch();
+ }
+
// Punt storage for now.
}
@@ -239,6 +277,11 @@ public class ResourceGroup {
this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, resourceUsage.getDispatch(),
broker);
}
+
+ if (resourceUsage.hasReplicationDispatch()) {
+ this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch,
+ resourceUsage.getReplicationDispatch(), broker);
+ }
// Punt storage for now.
}
@@ -319,7 +362,7 @@ public class ResourceGroup {
monEntity.usageFromOtherBrokersLock.lock();
try {
- pbus = monEntity.usageFromOtherBrokers.get(myBrokerId);
+ pbus = monEntity.usageFromOtherBrokers.getIfPresent(myBrokerId);
} finally {
monEntity.usageFromOtherBrokersLock.unlock();
}
@@ -343,7 +386,7 @@ public class ResourceGroup {
monEntity.usageFromOtherBrokersLock.lock();
BytesAndMessagesCount retStats = new BytesAndMessagesCount();
try {
- monEntity.usageFromOtherBrokers.forEach((broker, brokerUsage) -> {
+ monEntity.usageFromOtherBrokers.asMap().forEach((broker, brokerUsage) -> {
retStats.bytes += brokerUsage.usedValues.bytes;
retStats.messages += brokerUsage.usedValues.messages;
});
@@ -356,14 +399,6 @@ public class ResourceGroup {
protected BytesAndMessagesCount updateLocalQuota(ResourceGroupMonitoringClass monClass,
BytesAndMessagesCount newQuota) throws PulsarAdminException {
- // Only the Publish side is functional now; add the Dispatch side code when the consume side is ready.
- if (!ResourceGroupMonitoringClass.Publish.equals(monClass)) {
- if (log.isDebugEnabled()) {
- log.debug("Doing nothing for monClass={}; only Publish is functional", monClass);
- }
- return null;
- }
-
this.checkMonitoringClass(monClass);
BytesAndMessagesCount oldBMCount;
@@ -372,7 +407,22 @@ public class ResourceGroup {
oldBMCount = monEntity.quotaForNextPeriod;
try {
monEntity.quotaForNextPeriod = newQuota;
- this.resourceGroupPublishLimiter.update(newQuota);
+ switch (monClass) {
+ case ReplicationDispatch:
+ ResourceGroupRateLimiterManager
+ .updateReplicationDispatchRateLimiter(resourceGroupReplicationDispatchLimiter, newQuota);
+ break;
+ case Publish:
+ this.resourceGroupPublishLimiter.update(newQuota);
+ break;
+ case Dispatch:
+ ResourceGroupRateLimiterManager.updateDispatchRateLimiter(resourceGroupDispatchLimiter, newQuota);
+ break;
+ default:
+ if (log.isDebugEnabled()) {
+ log.debug("Doing nothing for monClass={};", monClass);
+ }
+ }
} finally {
monEntity.localUsageStatsLock.unlock();
}
@@ -424,9 +474,16 @@ public class ResourceGroup {
}
private void checkMonitoringClass(ResourceGroupMonitoringClass monClass) throws PulsarAdminException {
- if (monClass != ResourceGroupMonitoringClass.Publish && monClass != ResourceGroupMonitoringClass.Dispatch) {
- String errMesg = "Unexpected monitoring class: " + monClass;
- throw new PulsarAdminException(errMesg);
+ switch (monClass) {
+ case Publish:
+ break;
+ case Dispatch:
+ break;
+ case ReplicationDispatch:
+ break;
+ default:
+ String errMesg = "Unexpected monitoring class: " + monClass;
+ throw new PulsarAdminException(errMesg);
}
}
@@ -501,7 +558,7 @@ public class ResourceGroup {
long newByteCount, newMessageCount;
monEntity = this.monitoringClassFields[idx];
- usageStats = monEntity.usageFromOtherBrokers.get(broker);
+ usageStats = monEntity.usageFromOtherBrokers.getIfPresent(broker);
if (usageStats == null) {
usageStats = new PerBrokerUsageStats();
usageStats.usedValues = new BytesAndMessagesCount();
@@ -513,13 +570,15 @@ public class ResourceGroup {
newMessageCount = p.getMessagesPerPeriod();
usageStats.usedValues.messages = newMessageCount;
usageStats.lastResourceUsageReadTimeMSecsSinceEpoch = System.currentTimeMillis();
- oldUsageStats = monEntity.usageFromOtherBrokers.put(broker, usageStats);
+ oldUsageStats = monEntity.usageFromOtherBrokers.getIfPresent(broker);
+ monEntity.usageFromOtherBrokers.put(broker, usageStats);
} finally {
monEntity.usageFromOtherBrokersLock.unlock();
}
rgRemoteUsageReportsBytes.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newByteCount);
rgRemoteUsageReportsMessages.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newMessageCount);
-
+ rgRemoteUsageReportsBytesGauge.labels(this.ruConsumer.getID(), monClass.name(), broker).set(newByteCount);
+ rgRemoteUsageReportsMessagesGauge.labels(this.ruConsumer.getID(), monClass.name(), broker).set(newMessageCount);
oldByteCount = oldMessageCount = -1;
if (oldUsageStats != null) {
oldByteCount = oldUsageStats.usedValues.bytes;
@@ -536,21 +595,18 @@ public class ResourceGroup {
}
private void setResourceGroupMonitoringClassFields() {
- PerMonitoringClassFields monClassFields;
+ ServiceConfiguration conf = rgs.getPulsar().getConfiguration();
+ long resourceUsageTransportPublishIntervalInSecs = conf.getResourceUsageTransportPublishIntervalInSecs();
+ int maxUsageReportSuppressRounds = Math.max(conf.getResourceUsageMaxUsageReportSuppressRounds(), 1);
+ long cacheInMS = TimeUnit.SECONDS.toMillis(
+ resourceUsageTransportPublishIntervalInSecs * maxUsageReportSuppressRounds * 2);
+ // Usage report data is cached to the memory, when the broker is restart or offline, we need an elimination
+ // strategy to release the quota occupied by other broker.
+ //
+ // Considering that each broker starts at a different time, the cache time should be equal to the mandatory
+ // reporting period * 2.
for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) {
- this.monitoringClassFields[idx] = new PerMonitoringClassFields();
-
- monClassFields = this.monitoringClassFields[idx];
- monClassFields.configValuesPerPeriod = new BytesAndMessagesCount();
- monClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount();
- monClassFields.lastReportedValues = new BytesAndMessagesCount();
- monClassFields.quotaForNextPeriod = new BytesAndMessagesCount();
- monClassFields.totalUsedLocally = new BytesAndMessagesCount();
- monClassFields.usageFromOtherBrokers = new HashMap<>();
-
- monClassFields.usageFromOtherBrokersLock = new ReentrantLock();
- // ToDo: Change the following to a ReadWrite lock if needed.
- monClassFields.localUsageStatsLock = new ReentrantLock();
+ this.monitoringClassFields[idx] = PerMonitoringClassFields.create(cacheInMS);
}
}
@@ -568,6 +624,14 @@ public class ResourceGroup {
? -1 : rgConfig.getDispatchRateInBytes();
this.monitoringClassFields[idx].configValuesPerPeriod.messages = rgConfig.getDispatchRateInMsgs() == null
? -1 : rgConfig.getDispatchRateInMsgs();
+
+ idx = ResourceGroupMonitoringClass.ReplicationDispatch.ordinal();
+ this.monitoringClassFields[idx]
+ .configValuesPerPeriod.bytes = rgConfig.getReplicationDispatchRateInBytes() == null
+ ? -1 : rgConfig.getReplicationDispatchRateInBytes();
+ this.monitoringClassFields[idx]
+ .configValuesPerPeriod.messages = rgConfig.getReplicationDispatchRateInMsgs() == null
+ ? -1 : rgConfig.getReplicationDispatchRateInMsgs();
}
private void setDefaultResourceUsageTransportHandlers() {
@@ -613,6 +677,7 @@ public class ResourceGroup {
// across all of its usage classes (publish/dispatch/...).
private Set<String> resourceGroupTenantRefs = ConcurrentHashMap.newKeySet();
private Set<String> resourceGroupNamespaceRefs = ConcurrentHashMap.newKeySet();
+ private Set<String> resourceGroupTopicRefs = ConcurrentHashMap.newKeySet();
// Blobs required for transport manager's resource-usage register/unregister ops.
ResourceUsageConsumer ruConsumer;
@@ -631,11 +696,21 @@ public class ResourceGroup {
.help("Bytes used reported about this <RG, monitoring class> from a remote broker")
.labelNames(resourceGroupMontoringclassRemotebrokerLabels)
.register();
+ private static final Gauge rgRemoteUsageReportsBytesGauge = Gauge.build()
+ .name("pulsar_resource_group_remote_usage_bytes_used_gauge")
+ .help("Bytes used reported about this <RG, monitoring class> from a remote broker")
+ .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
+ .register();
private static final Counter rgRemoteUsageReportsMessages = Counter.build()
.name("pulsar_resource_group_remote_usage_messages_used")
.help("Messages used reported about this <RG, monitoring class> from a remote broker")
.labelNames(resourceGroupMontoringclassRemotebrokerLabels)
.register();
+ private static final Gauge rgRemoteUsageReportsMessagesGauge = Gauge.build()
+ .name("pulsar_resource_group_remote_usage_messages_used_gauge")
+ .help("Messages used reported about this <RG, monitoring class> from a remote broker")
+ .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
+ .register();
private static final Counter rgLocalUsageReportCount = Counter.build()
.name("pulsar_resource_group_local_usage_reported")
@@ -647,6 +722,12 @@ public class ResourceGroup {
@Getter
protected ResourceGroupPublishLimiter resourceGroupPublishLimiter;
+ @Getter
+ protected ResourceGroupDispatchLimiter resourceGroupReplicationDispatchLimiter;
+
+ @Getter
+ protected ResourceGroupDispatchLimiter resourceGroupDispatchLimiter;
+
protected static class PerMonitoringClassFields {
// This lock covers all the "local" counts (i.e., except for the per-broker usage stats).
Lock localUsageStatsLock;
@@ -671,11 +752,33 @@ public class ResourceGroup {
int numSuppressedUsageReports;
// Accumulated stats of local usage.
+ @VisibleForTesting
BytesAndMessagesCount totalUsedLocally;
// This lock covers all the non-local usage counts, received from other brokers.
Lock usageFromOtherBrokersLock;
- public HashMap<String, PerBrokerUsageStats> usageFromOtherBrokers;
+ public Cache<String, PerBrokerUsageStats> usageFromOtherBrokers;
+
+ private PerMonitoringClassFields(){
+
+ }
+
+ static PerMonitoringClassFields create(long durationMs) {
+ PerMonitoringClassFields perMonitoringClassFields = new PerMonitoringClassFields();
+ perMonitoringClassFields.configValuesPerPeriod = new BytesAndMessagesCount();
+ perMonitoringClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount();
+ perMonitoringClassFields.lastReportedValues = new BytesAndMessagesCount();
+ perMonitoringClassFields.quotaForNextPeriod = new BytesAndMessagesCount();
+ perMonitoringClassFields.totalUsedLocally = new BytesAndMessagesCount();
+ perMonitoringClassFields.usageFromOtherBrokersLock = new ReentrantLock();
+ // ToDo: Change the following to a ReadWrite lock if needed.
+ perMonitoringClassFields.localUsageStatsLock = new ReentrantLock();
+
+ perMonitoringClassFields.usageFromOtherBrokers = Caffeine.newBuilder()
+ .expireAfterWrite(durationMs, TimeUnit.MILLISECONDS)
+ .build();
+ return perMonitoringClassFields;
+ }
}
// Usage stats for this RG obtained from other brokers.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java
new file mode 100644
index 00000000000..db2a9045511
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.common.util.RateLimiter;
+
+public class ResourceGroupDispatchLimiter implements AutoCloseable {
+
+ private final ScheduledExecutorService executorService;
+ private volatile RateLimiter dispatchRateLimiterOnMessage;
+ private volatile RateLimiter dispatchRateLimiterOnByte;
+
+ public ResourceGroupDispatchLimiter(ScheduledExecutorService executorService,
+ long dispatchRateInMsgs, long dispatchRateInBytes) {
+ this.executorService = executorService;
+ update(dispatchRateInMsgs, dispatchRateInBytes);
+ }
+
+ public void update(long dispatchRateInMsgs, long dispatchRateInBytes) {
+ if (dispatchRateInMsgs > 0) {
+ if (dispatchRateLimiterOnMessage != null) {
+ this.dispatchRateLimiterOnMessage.setRate(dispatchRateInMsgs);
+ } else {
+ this.dispatchRateLimiterOnMessage =
+ RateLimiter.builder()
+ .scheduledExecutorService(executorService)
+ .permits(dispatchRateInMsgs)
+ .rateTime(1)
+ .timeUnit(TimeUnit.SECONDS)
+ .permitUpdater(null)
+ .isDispatchOrPrecisePublishRateLimiter(true)
+ .build();
+ }
+ } else {
+ if (this.dispatchRateLimiterOnMessage != null) {
+ this.dispatchRateLimiterOnMessage.close();
+ this.dispatchRateLimiterOnMessage = null;
+ }
+ }
+
+ if (dispatchRateInBytes > 0) {
+ if (dispatchRateLimiterOnByte != null) {
+ this.dispatchRateLimiterOnByte.setRate(dispatchRateInBytes);
+ } else {
+ this.dispatchRateLimiterOnByte =
+ RateLimiter.builder()
+ .scheduledExecutorService(executorService)
+ .permits(dispatchRateInBytes)
+ .rateTime(1)
+ .timeUnit(TimeUnit.SECONDS)
+ .permitUpdater(null)
+ .isDispatchOrPrecisePublishRateLimiter(true)
+ .build();
+ }
+ } else {
+ if (this.dispatchRateLimiterOnByte != null) {
+ this.dispatchRateLimiterOnByte.close();
+ this.dispatchRateLimiterOnByte = null;
+ }
+ }
+ }
+
+ /**
+ * returns available msg-permit if msg-dispatch-throttling is enabled else it returns -1.
+ *
+ * @return
+ */
+ public long getAvailableDispatchRateLimitOnMsg() {
+ return dispatchRateLimiterOnMessage == null ? -1 : dispatchRateLimiterOnMessage.getAvailablePermits();
+ }
+
+ /**
+ * returns available byte-permit if msg-dispatch-throttling is enabled else it returns -1.
+ *
+ * @return
+ */
+ public long getAvailableDispatchRateLimitOnByte() {
+ return dispatchRateLimiterOnByte == null ? -1 : dispatchRateLimiterOnByte.getAvailablePermits();
+ }
+
+ /**
+ * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed.
+ *
+ * @param numberOfMessages
+ * @param byteSize
+ */
+ public void consumeDispatchQuota(long numberOfMessages, long byteSize) {
+ if (numberOfMessages > 0 && dispatchRateLimiterOnMessage != null) {
+ dispatchRateLimiterOnMessage.tryAcquire(numberOfMessages);
+ }
+ if (byteSize > 0 && dispatchRateLimiterOnByte != null) {
+ dispatchRateLimiterOnByte.tryAcquire(byteSize);
+ }
+ }
+
+ /**
+ * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed.
+ *
+ * @param numberOfMessages
+ * @param byteSize
+ */
+ public boolean tryAcquire(long numberOfMessages, long byteSize) {
+ if (numberOfMessages > 0 && dispatchRateLimiterOnMessage != null) {
+ if (!dispatchRateLimiterOnMessage.tryAcquire(numberOfMessages)) {
+ return false;
+ }
+ }
+ if (byteSize > 0 && dispatchRateLimiterOnByte != null) {
+ if (!dispatchRateLimiterOnByte.tryAcquire(byteSize)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Checks if dispatch-rate limiting is enabled.
+ *
+ * @return
+ */
+ public boolean isDispatchRateLimitingEnabled() {
+ return dispatchRateLimiterOnMessage != null || dispatchRateLimiterOnByte != null;
+ }
+
+ public void close() {
+ if (dispatchRateLimiterOnMessage != null) {
+ dispatchRateLimiterOnMessage = null;
+ }
+ if (dispatchRateLimiterOnByte != null) {
+ dispatchRateLimiterOnByte = null;
+ }
+ }
+
+ /**
+ * Get configured msg dispatch-throttling rate. Returns -1 if not configured
+ *
+ * @return
+ */
+ public long getDispatchRateOnMsg() {
+ return dispatchRateLimiterOnMessage != null ? dispatchRateLimiterOnMessage.getRate() : -1;
+ }
+
+ /**
+ * Get configured byte dispatch-throttling rate. Returns -1 if not configured
+ *
+ * @return
+ */
+ public long getDispatchRateOnByte() {
+ return dispatchRateLimiterOnByte != null ? dispatchRateLimiterOnByte.getRate() : -1;
+ }
+
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java
new file mode 100644
index 00000000000..2011ad12974
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import java.util.Optional;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+
+public class ResourceGroupRateLimiterManager {
+
+ static ResourceGroupDispatchLimiter newReplicationDispatchRateLimiter(
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup,
+ ScheduledExecutorService executorService) {
+ long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L);
+ long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L);
+ return new ResourceGroupDispatchLimiter(executorService, msgs, bytes);
+ }
+
+ static void updateReplicationDispatchRateLimiter(
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) {
+ long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L);
+ long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L);
+ resourceGroupDispatchLimiter.update(msgs, bytes);
+ }
+
+ static void updateReplicationDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+ BytesAndMessagesCount quota) {
+ resourceGroupDispatchLimiter.update(quota.messages, quota.bytes);
+ }
+
+ static ResourceGroupDispatchLimiter newDispatchRateLimiter(
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup,
+ ScheduledExecutorService executorService) {
+ long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1);
+ long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L);
+ return new ResourceGroupDispatchLimiter(executorService, msgs, bytes);
+ }
+
+ static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) {
+ long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1);
+ long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L);
+ resourceGroupDispatchLimiter.update(msgs, bytes);
+ }
+
+ static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+ BytesAndMessagesCount quota) {
+ resourceGroupDispatchLimiter.update(quota.messages, quota.bytes);
+ }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
index 1f4b3f32cb9..c4a774aa28b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
@@ -19,14 +19,22 @@
package org.apache.pulsar.broker.resourcegroup;
import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import io.prometheus.client.Counter;
+import io.prometheus.client.Gauge;
import io.prometheus.client.Summary;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
import lombok.Getter;
import lombok.val;
import org.apache.pulsar.broker.PulsarService;
@@ -60,7 +68,7 @@ public class ResourceGroupService implements AutoCloseable{
public ResourceGroupService(PulsarService pulsar) {
this.pulsar = pulsar;
this.timeUnitScale = TimeUnit.SECONDS;
- this.quotaCalculator = new ResourceQuotaCalculatorImpl();
+ this.quotaCalculator = new ResourceQuotaCalculatorImpl(pulsar);
this.resourceUsageTransportManagerMgr = pulsar.getResourceUsageTransportManager();
this.rgConfigListener = new ResourceGroupConfigListener(this, pulsar);
this.initialize();
@@ -167,9 +175,11 @@ public class ResourceGroupService implements AutoCloseable{
long tenantRefCount = rg.getResourceGroupNumOfTenantRefs();
long nsRefCount = rg.getResourceGroupNumOfNSRefs();
- if ((tenantRefCount + nsRefCount) > 0) {
+ long topicRefCount = rg.getResourceGroupNumOfTopicRefs();
+ if ((tenantRefCount + nsRefCount + topicRefCount) > 0) {
String errMesg = "Resource group " + name + " still has " + tenantRefCount + " tenant refs";
errMesg += " and " + nsRefCount + " namespace refs on it";
+ errMesg += " and " + topicRefCount + " topic refs on it";
throw new PulsarAdminException(errMesg);
}
@@ -288,11 +298,92 @@ public class ResourceGroupService implements AutoCloseable{
throw new PulsarAdminException(errMesg);
}
+ aggregateLock.lock();
+
+ Set<String> invalidateAllKeyForProduce = new HashSet<>();
+ topicProduceStats.asMap().forEach((key, value) -> {
+ TopicName topicName = TopicName.get(key);
+ if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+ invalidateAllKeyForProduce.add(key);
+ }
+ });
+ topicProduceStats.invalidateAll(invalidateAllKeyForProduce);
+
+ Set<String> invalidateAllKeyForReplication = new HashSet<>();
+ topicToReplicatorsMap.forEach((key, value) -> {
+ TopicName topicName = TopicName.get(key);
+ if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+ topicToReplicatorsMap.remove(key);
+ value.forEach(n -> invalidateAllKeyForReplication.add(getReplicatorKey(topicName.toString(), n)));
+ }
+ });
+ replicationDispatchStats.invalidateAll(invalidateAllKeyForReplication);
+
+ Set<String> invalidateAllKeyForConsumer = new HashSet<>();
+ topicConsumeStats.asMap().forEach((key, value) -> {
+ TopicName topicName = TopicName.get(key);
+ if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+ invalidateAllKeyForConsumer.add(key);
+ }
+ });
+ topicConsumeStats.invalidateAll(invalidateAllKeyForConsumer);
+
+ aggregateLock.unlock();
// Dissociate this NS-name from the RG.
this.namespaceToRGsMap.remove(fqNamespaceName, rg);
rgNamespaceUnRegisters.labels(resourceGroupName).inc();
}
+ /**
+ * Registers a topic as a user of a resource group.
+ *
+ * @param resourceGroupName
+ * @param topicName complete topic name
+ */
+ public void registerTopic(String resourceGroupName, TopicName topicName) {
+ ResourceGroup rg = resourceGroupsMap.get(resourceGroupName);
+ if (rg == null) {
+ throw new IllegalStateException("Resource group does not exist: " + resourceGroupName);
+ }
+
+ ResourceGroupOpStatus status = rg.registerUsage(topicName.toString(), ResourceGroupRefTypes.Topics,
+ true, this.resourceUsageTransportManagerMgr);
+ if (status == ResourceGroupOpStatus.Exists) {
+ String msg = String.format("Topic %s already references the target resource group %s",
+ topicName, resourceGroupName);
+ throw new IllegalStateException(msg);
+ }
+
+ // Associate this topic-name with the RG.
+ this.topicToRGsMap.put(topicName, rg);
+ rgTopicRegisters.labels(resourceGroupName).inc();
+ }
+
+ /**
+ * UnRegisters a topic from a resource group.
+ *
+ * @param topicName complete topic name
+ */
+ public void unRegisterTopic(TopicName topicName) {
+ aggregateLock.lock();
+ String topicNameString = topicName.toString();
+ ResourceGroup remove = topicToRGsMap.remove(topicName);
+ if (remove != null) {
+ remove.registerUsage(topicNameString, ResourceGroupRefTypes.Topics,
+ false, this.resourceUsageTransportManagerMgr);
+ rgTopicUnRegisters.labels(remove.resourceGroupName).inc();
+ }
+ topicProduceStats.invalidate(topicNameString);
+ topicConsumeStats.invalidate(topicNameString);
+ Set<String> replicators = topicToReplicatorsMap.remove(topicNameString);
+ if (replicators != null) {
+ List<String> keys = replicators.stream().map(n -> getReplicatorKey(topicNameString, n))
+ .collect(Collectors.toList());
+ replicationDispatchStats.invalidateAll(keys);
+ }
+ aggregateLock.unlock();
+ }
+
/**
* Return the resource group associated with a namespace.
*
@@ -303,6 +394,11 @@ public class ResourceGroupService implements AutoCloseable{
return this.namespaceToRGsMap.get(namespaceName);
}
+ @VisibleForTesting
+ public ResourceGroup getTopicResourceGroup(TopicName topicName) {
+ return this.topicToRGsMap.get(topicName);
+ }
+
@Override
public void close() throws Exception {
if (aggregateLocalUsagePeriodicTask != null) {
@@ -314,12 +410,21 @@ public class ResourceGroupService implements AutoCloseable{
resourceGroupsMap.clear();
tenantToRGsMap.clear();
namespaceToRGsMap.clear();
- topicProduceStats.clear();
- topicConsumeStats.clear();
+ topicProduceStats.invalidateAll();
+ topicConsumeStats.invalidateAll();
+ replicationDispatchStats.invalidateAll();
+ }
+
+ private void incrementUsage(ResourceGroup resourceGroup,
+ ResourceGroupMonitoringClass monClass, BytesAndMessagesCount incStats)
+ throws PulsarAdminException {
+ resourceGroup.incrementLocalUsageStats(monClass, incStats);
+ rgLocalUsageBytes.labels(resourceGroup.resourceGroupName, monClass.name()).inc(incStats.bytes);
+ rgLocalUsageMessages.labels(resourceGroup.resourceGroupName, monClass.name()).inc(incStats.messages);
}
/**
- * Increments usage stats for the resource groups associated with the given namespace and tenant.
+ * Increments usage stats for the resource groups associated with the given namespace, tenant, and topic.
* Expected to be called when a message is produced or consumed on a topic, or when we calculate
* usage periodically in the background by going through broker-service stats. [Not yet decided
* which model we will follow.] Broker-service stats will be cumulative, while calls from the
@@ -327,22 +432,25 @@ public class ResourceGroupService implements AutoCloseable{
*
* If the tenant and NS are associated with different RGs, the statistics on both RGs are updated.
* If the tenant and NS are associated with the same RG, the stats on the RG are updated only once
+ * If the tenant, NS and topic are associated with the same RG, the stats on the RG are updated only once
* (to avoid a direct double-counting).
* ToDo: will this distinction result in "expected semantics", or shock from users?
* For now, the only caller is internal to this class.
*
* @param tenantName
- * @param nsName
+ * @param nsName Complete namespace name
+ * @param topicName Complete topic name
* @param monClass
* @param incStats
* @returns true if the stats were updated; false if nothing was updated.
*/
- protected boolean incrementUsage(String tenantName, String nsName,
- ResourceGroupMonitoringClass monClass,
- BytesAndMessagesCount incStats) throws PulsarAdminException {
- final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(tenantName, nsName));
+ protected boolean incrementUsage(String tenantName, String nsName, String topicName,
+ ResourceGroupMonitoringClass monClass,
+ BytesAndMessagesCount incStats) throws PulsarAdminException {
+ final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(nsName));
final ResourceGroup tenantRG = this.tenantToRGsMap.get(tenantName);
- if (tenantRG == null && nsRG == null) {
+ final ResourceGroup topicRG = this.topicToRGsMap.get(TopicName.get(topicName));
+ if (tenantRG == null && nsRG == null && topicRG == null) {
return false;
}
@@ -353,24 +461,40 @@ public class ResourceGroupService implements AutoCloseable{
throw new PulsarAdminException(errMesg);
}
- if (nsRG == tenantRG) {
+ if (tenantRG == nsRG && nsRG == topicRG) {
// Update only once in this case.
- // Note that we will update both tenant and namespace RGs in other cases.
- nsRG.incrementLocalUsageStats(monClass, incStats);
- rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages);
- rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+ // Note that we will update both tenant, namespace and topic RGs in other cases.
+ incrementUsage(tenantRG, monClass, incStats);
+ return true;
+ }
+
+ if (tenantRG != null && tenantRG == nsRG) {
+ // Tenant and Namespace GRs are same.
+ incrementUsage(tenantRG, monClass, incStats);
+ if (topicRG == null) {
+ return true;
+ }
+ }
+
+ if (nsRG != null && nsRG == topicRG) {
+ // Namespace and Topic GRs are same.
+ incrementUsage(nsRG, monClass, incStats);
return true;
}
if (tenantRG != null) {
- tenantRG.incrementLocalUsageStats(monClass, incStats);
- rgLocalUsageMessages.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.messages);
- rgLocalUsageBytes.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+ // Tenant GR is different from other resource GR.
+ incrementUsage(tenantRG, monClass, incStats);
}
+
if (nsRG != null) {
- nsRG.incrementLocalUsageStats(monClass, incStats);
- rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages);
- rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+ // Namespace GR is different from other resource GR.
+ incrementUsage(nsRG, monClass, incStats);
+ }
+
+ if (topicRG != null) {
+ // Topic GR is different from other resource GR.
+ incrementUsage(topicRG, monClass, incStats);
}
return true;
@@ -419,11 +543,17 @@ public class ResourceGroupService implements AutoCloseable{
return rg;
}
+ private String getReplicatorKey(String topic, String replicationRemoteCluster) {
+ return topic + replicationRemoteCluster;
+ }
+
// Find the difference between the last time stats were updated for this topic, and the current
// time. If the difference is positive, update the stats.
- private void updateStatsWithDiff(String topicName, String tenantString, String nsString,
- long accByteCount, long accMesgCount, ResourceGroupMonitoringClass monClass) {
- ConcurrentHashMap<String, BytesAndMessagesCount> hm;
+ @VisibleForTesting
+ protected void updateStatsWithDiff(String topicName, String replicationRemoteCluster, String tenantString,
+ String nsString, long accByteCount, long accMsgCount,
+ ResourceGroupMonitoringClass monClass) {
+ Cache<String, BytesAndMessagesCount> hm;
switch (monClass) {
default:
log.error("updateStatsWithDiff: Unknown monitoring class={}; ignoring", monClass);
@@ -436,6 +566,10 @@ public class ResourceGroupService implements AutoCloseable{
case Dispatch:
hm = this.topicConsumeStats;
break;
+
+ case ReplicationDispatch:
+ hm = this.replicationDispatchStats;
+ break;
}
BytesAndMessagesCount bmDiff = new BytesAndMessagesCount();
@@ -443,9 +577,22 @@ public class ResourceGroupService implements AutoCloseable{
BytesAndMessagesCount bmNewCount = new BytesAndMessagesCount();
bmNewCount.bytes = accByteCount;
- bmNewCount.messages = accMesgCount;
-
- bmOldCount = hm.get(topicName);
+ bmNewCount.messages = accMsgCount;
+
+ String key;
+ if (monClass == ResourceGroupMonitoringClass.ReplicationDispatch) {
+ key = getReplicatorKey(topicName, replicationRemoteCluster);
+ topicToReplicatorsMap.compute(topicName, (n, value) -> {
+ if (value == null) {
+ value = new CopyOnWriteArraySet<>();
+ }
+ value.add(replicationRemoteCluster);
+ return value;
+ });
+ } else {
+ key = topicName;
+ }
+ bmOldCount = hm.getIfPresent(key);
if (bmOldCount == null) {
bmDiff.bytes = bmNewCount.bytes;
bmDiff.messages = bmNewCount.messages;
@@ -459,14 +606,14 @@ public class ResourceGroupService implements AutoCloseable{
}
try {
- boolean statsUpdated = this.incrementUsage(tenantString, nsString, monClass, bmDiff);
+ boolean statsUpdated = this.incrementUsage(tenantString, nsString, topicName, monClass, bmDiff);
if (log.isDebugEnabled()) {
log.debug("updateStatsWithDiff for topic={}: monclass={} statsUpdated={} for tenant={}, namespace={}; "
+ "by {} bytes, {} mesgs",
topicName, monClass, statsUpdated, tenantString, nsString,
bmDiff.bytes, bmDiff.messages);
}
- hm.put(topicName, bmNewCount);
+ hm.put(key, bmNewCount);
} catch (Throwable t) {
log.error("updateStatsWithDiff: got ex={} while aggregating for {} side",
t.getMessage(), monClass);
@@ -488,11 +635,21 @@ public class ResourceGroupService implements AutoCloseable{
return rgCalculatedQuotaBytes.labels(rgName, monClassName).get();
}
+ // Visibility for testing.
+ protected static double getRgQuotaByte (String rgName, String monClassName) {
+ return rgCalculatedQuotaBytesGauge.labels(rgName, monClassName).get();
+ }
+
// Visibility for testing.
protected static double getRgQuotaMessageCount (String rgName, String monClassName) {
return rgCalculatedQuotaMessages.labels(rgName, monClassName).get();
}
+ // Visibility for testing.
+ protected static double getRgQuotaMessage(String rgName, String monClassName) {
+ return rgCalculatedQuotaMessagesGauge.labels(rgName, monClassName).get();
+ }
+
// Visibility for testing.
protected static double getRgLocalUsageByteCount (String rgName, String monClassName) {
return rgLocalUsageBytes.labels(rgName, monClassName).get();
@@ -545,30 +702,40 @@ public class ResourceGroupService implements AutoCloseable{
BrokerService bs = this.pulsar.getBrokerService();
Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
+ aggregateLock.lock();
for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
final String topicName = entry.getKey();
final TopicStats topicStats = entry.getValue();
final TopicName topic = TopicName.get(topicName);
final String tenantString = topic.getTenant();
- final String nsString = topic.getNamespacePortion();
+ final String nsString = topic.getNamespace();
final NamespaceName fqNamespace = topic.getNamespaceObject();
// Can't use containsKey here, as that checks for exact equality
// (we need a check for string-comparison).
val tenantRG = this.tenantToRGsMap.get(tenantString);
val namespaceRG = this.namespaceToRGsMap.get(fqNamespace);
- if (tenantRG == null && namespaceRG == null) {
+ val topicRG = this.topicToRGsMap.get(topic);
+ if (tenantRG == null && namespaceRG == null && topicRG == null) {
// This topic's NS/tenant are not registered to any RG.
continue;
}
- this.updateStatsWithDiff(topicName, tenantString, nsString,
+ topicStats.getReplication().forEach((remoteCluster, stats) -> {
+ this.updateStatsWithDiff(topicName, remoteCluster, tenantString, nsString,
+ stats.getBytesOutCounter(),
+ stats.getMsgOutCounter(),
+ ResourceGroupMonitoringClass.ReplicationDispatch
+ );
+ });
+ this.updateStatsWithDiff(topicName, null, tenantString, nsString,
topicStats.getBytesInCounter(), topicStats.getMsgInCounter(),
ResourceGroupMonitoringClass.Publish);
- this.updateStatsWithDiff(topicName, tenantString, nsString,
+ this.updateStatsWithDiff(topicName, null, tenantString, nsString,
topicStats.getBytesOutCounter(), topicStats.getMsgOutCounter(),
ResourceGroupMonitoringClass.Dispatch);
}
+ aggregateLock.unlock();
double diffTimeSeconds = aggrUsageTimer.observeDuration();
if (log.isDebugEnabled()) {
log.debug("aggregateResourceGroupLocalUsages took {} milliseconds", diffTimeSeconds * 1000);
@@ -629,13 +796,7 @@ public class ResourceGroupService implements AutoCloseable{
globUsageMessagesArray);
BytesAndMessagesCount oldBMCount = resourceGroup.updateLocalQuota(monClass, updatedQuota);
- // Guard against unconfigured quota settings, for which computeLocalQuota will return negative.
- if (updatedQuota.messages >= 0) {
- rgCalculatedQuotaMessages.labels(rgName, monClass.name()).inc(updatedQuota.messages);
- }
- if (updatedQuota.bytes >= 0) {
- rgCalculatedQuotaBytes.labels(rgName, monClass.name()).inc(updatedQuota.bytes);
- }
+ incRgCalculatedQuota(rgName, monClass, updatedQuota, resourceUsagePublishPeriodInSeconds);
if (oldBMCount != null) {
long messagesIncrement = updatedQuota.messages - oldBMCount.messages;
long bytesIncrement = updatedQuota.bytes - oldBMCount.bytes;
@@ -686,8 +847,6 @@ public class ResourceGroupService implements AutoCloseable{
newPeriodInSeconds,
timeUnitScale);
this.resourceUsagePublishPeriodInSeconds = newPeriodInSeconds;
- maxIntervalForSuppressingReportsMSecs =
- TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds;
}
}
@@ -705,9 +864,11 @@ public class ResourceGroupService implements AutoCloseable{
periodInSecs,
periodInSecs,
this.timeUnitScale);
- maxIntervalForSuppressingReportsMSecs =
- TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds;
-
+ long resourceUsagePublishPeriodInMS = TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds);
+ long statsCacheInMS = resourceUsagePublishPeriodInMS * 2;
+ topicProduceStats = newStatsCache(statsCacheInMS);
+ topicConsumeStats = newStatsCache(statsCacheInMS);
+ replicationDispatchStats = newStatsCache(statsCacheInMS);
}
private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig)
@@ -726,6 +887,28 @@ public class ResourceGroupService implements AutoCloseable{
}
}
+ static void incRgCalculatedQuota(String rgName, ResourceGroupMonitoringClass monClass,
+ BytesAndMessagesCount updatedQuota, long resourceUsagePublishPeriodInSeconds) {
+ // Guard against unconfigured quota settings, for which computeLocalQuota will return negative.
+ if (updatedQuota.messages >= 0) {
+ rgCalculatedQuotaMessages.labels(rgName, monClass.name())
+ .inc(updatedQuota.messages * resourceUsagePublishPeriodInSeconds);
+ rgCalculatedQuotaMessagesGauge.labels(rgName, monClass.name()).set(updatedQuota.messages);
+ }
+ if (updatedQuota.bytes >= 0) {
+ rgCalculatedQuotaBytes.labels(rgName, monClass.name())
+ .inc(updatedQuota.bytes * resourceUsagePublishPeriodInSeconds);
+ rgCalculatedQuotaBytesGauge.labels(rgName, monClass.name()).set(updatedQuota.bytes);
+ }
+ }
+
+ @VisibleForTesting
+ protected Cache<String, BytesAndMessagesCount> newStatsCache(long durationMS) {
+ return Caffeine.newBuilder()
+ .expireAfterAccess(durationMS, TimeUnit.MILLISECONDS)
+ .build();
+ }
+
private static final Logger log = LoggerFactory.getLogger(ResourceGroupService.class);
@Getter
@@ -745,11 +928,14 @@ public class ResourceGroupService implements AutoCloseable{
// Given a qualified NS-name (i.e., in "tenant/namespace" format), record its associated resource-group
private ConcurrentHashMap<NamespaceName, ResourceGroup> namespaceToRGsMap = new ConcurrentHashMap<>();
+ private ConcurrentHashMap<TopicName, ResourceGroup> topicToRGsMap = new ConcurrentHashMap<>();
+ private ConcurrentHashMap<String, Set<String>> topicToReplicatorsMap = new ConcurrentHashMap<>();
+ private ReentrantLock aggregateLock = new ReentrantLock();
// Maps to maintain the usage per topic, in produce/consume directions.
- private ConcurrentHashMap<String, BytesAndMessagesCount> topicProduceStats = new ConcurrentHashMap<>();
- private ConcurrentHashMap<String, BytesAndMessagesCount> topicConsumeStats = new ConcurrentHashMap<>();
-
+ private Cache<String, BytesAndMessagesCount> topicProduceStats;
+ private Cache<String, BytesAndMessagesCount> topicConsumeStats;
+ private Cache<String, BytesAndMessagesCount> replicationDispatchStats;
// The task that periodically re-calculates the quota budget for local usage.
private ScheduledFuture<?> aggregateLocalUsagePeriodicTask;
@@ -762,22 +948,6 @@ public class ResourceGroupService implements AutoCloseable{
// Allow a pluggable scale on time units; for testing periodic functionality.
private TimeUnit timeUnitScale;
- // The maximum number of successive rounds that we can suppress reporting local usage, because there was no
- // substantial change from the prior round. This is to ensure the reporting does not become too chatty.
- // Set this value to one more than the cadence of sending reports; e.g., if you want to send every 3rd report,
- // set the value to 4.
- // Setting this to 0 will make us report in every round.
- // Don't set to negative values; behavior will be "undefined".
- protected static final int MaxUsageReportSuppressRounds = 5;
-
- // Convenient shorthand, for MaxUsageReportSuppressRounds converted to a time interval in milliseconds.
- protected static long maxIntervalForSuppressingReportsMSecs;
-
- // The percentage difference that is considered "within limits" to suppress usage reporting.
- // Setting this to 0 will also make us report in every round.
- // Don't set it to negative values; behavior will be "undefined".
- protected static final float UsageReportSuppressionTolerancePercentage = 5;
-
// Labels for the various counters used here.
private static final String[] resourceGroupLabel = {"ResourceGroup"};
private static final String[] resourceGroupMonitoringclassLabels = {"ResourceGroup", "MonitoringClass"};
@@ -787,11 +957,21 @@ public class ResourceGroupService implements AutoCloseable{
.help("Bytes quota calculated for resource group")
.labelNames(resourceGroupMonitoringclassLabels)
.register();
+ private static final Gauge rgCalculatedQuotaBytesGauge = Gauge.build()
+ .name("pulsar_resource_group_calculated_bytes_quota_gauge")
+ .help("Bytes quota calculated for resource group")
+ .labelNames(resourceGroupMonitoringclassLabels)
+ .register();
private static final Counter rgCalculatedQuotaMessages = Counter.build()
.name("pulsar_resource_group_calculated_messages_quota")
.help("Messages quota calculated for resource group")
.labelNames(resourceGroupMonitoringclassLabels)
.register();
+ private static final Gauge rgCalculatedQuotaMessagesGauge = Gauge.build()
+ .name("pulsar_resource_group_calculated_messages_quota_gauge")
+ .help("Messages quota calculated for resource group")
+ .labelNames(resourceGroupMonitoringclassLabels)
+ .register();
private static final Counter rgLocalUsageBytes = Counter.build()
.name("pulsar_resource_group_bytes_used")
@@ -832,6 +1012,17 @@ public class ResourceGroupService implements AutoCloseable{
.labelNames(resourceGroupLabel)
.register();
+ private static final Counter rgTopicRegisters = Counter.build()
+ .name("pulsar_resource_group_topic_registers")
+ .help("Number of registrations of topics")
+ .labelNames(resourceGroupLabel)
+ .register();
+ private static final Counter rgTopicUnRegisters = Counter.build()
+ .name("pulsar_resource_group_topic_unregisters")
+ .help("Number of un-registrations of topics")
+ .labelNames(resourceGroupLabel)
+ .register();
+
private static final Summary rgUsageAggregationLatency = Summary.build()
.quantile(0.5, 0.05)
.quantile(0.9, 0.01)
@@ -847,15 +1038,25 @@ public class ResourceGroupService implements AutoCloseable{
.register();
@VisibleForTesting
- ConcurrentHashMap getTopicConsumeStats() {
+ Cache<String, BytesAndMessagesCount> getTopicConsumeStats() {
return this.topicConsumeStats;
}
@VisibleForTesting
- ConcurrentHashMap getTopicProduceStats() {
+ Cache<String, BytesAndMessagesCount> getTopicProduceStats() {
return this.topicProduceStats;
}
+ @VisibleForTesting
+ Cache<String, BytesAndMessagesCount> getReplicationDispatchStats() {
+ return this.replicationDispatchStats;
+ }
+
+ @VisibleForTesting
+ ConcurrentHashMap<String, Set<String>> getTopicToReplicatorsMap() {
+ return this.topicToReplicatorsMap;
+ }
+
@VisibleForTesting
ScheduledFuture<?> getAggregateLocalUsagePeriodicTask() {
return this.aggregateLocalUsagePeriodicTask;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
index 41291100a0d..fe1808077d7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
@@ -18,14 +18,22 @@
*/
package org.apache.pulsar.broker.resourcegroup;
-import static java.lang.Float.max;
import static java.lang.Math.abs;
+import java.util.concurrent.TimeUnit;
import lombok.val;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
+ private final PulsarService pulsarService;
+
+ public ResourceQuotaCalculatorImpl(PulsarService pulsarService) {
+ this.pulsarService = pulsarService;
+ }
+
@Override
public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) throws PulsarAdminException {
// ToDo: work out the initial conditions: we may allow a small number of "first few iterations" to go
@@ -53,13 +61,14 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
throw new PulsarAdminException(errMesg);
}
- // If the total usage is zero (which may happen during initial transients), just return the configured value.
+ // If the total/myUsage usage is zero (which may happen during initial transients, or when there is no
+ // traffic in the current broker), just return the configured value.
// The caller is expected to check the value returned, or not call here with a zero global usage.
// [This avoids a division by zero when calculating the local share.]
- if (totalUsage == 0) {
+ if (myUsage == 0 || totalUsage == 0) {
if (log.isDebugEnabled()) {
- log.debug("computeLocalQuota: totalUsage is zero; "
- + "returning the configured usage ({}) as new local quota",
+ log.debug("computeLocalQuota: totalUsage or myUsage is zero; "
+ + "returning the configured usage ({}) as new local quota",
confUsage);
}
return confUsage;
@@ -72,16 +81,25 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
log.warn(errMesg);
}
- // How much unused capacity is left over?
- float residual = confUsage - totalUsage;
+ int resourceUsageTransportPublishIntervalInSecs =
+ pulsarService.getConfiguration().getResourceUsageTransportPublishIntervalInSecs();
+ double resourceGroupLocalQuotaThreshold =
+ pulsarService.getConfiguration().getResourceGroupLocalQuotaThreshold();
+ float totalRate = (float) totalUsage / resourceUsageTransportPublishIntervalInSecs;
+ double adjustedConfUsage = confUsage * resourceGroupLocalQuotaThreshold;
+ if (totalRate <= adjustedConfUsage) {
+ log.info("computeLocalQuota: total usage ({}) is less than the "
+ + "configured usage * threshold ({}), "
+ + "returning the configured usage ({}) as new local quota",
+ totalRate, adjustedConfUsage, confUsage);
+ return confUsage;
+ }
// New quota is the old usage incremented by any residual as a ratio of the local usage to the total usage.
// This should result in the calculatedQuota increasing proportionately if total usage is less than the
// configured usage, and reducing proportionately if the total usage is greater than the configured usage.
- // Capped to 1, to prevent negative or zero setting of quota.
- // the rate limiter code assumes that rate value of 0 or less to mean that no rate limit should be applied
float myUsageFraction = (float) myUsage / totalUsage;
- float calculatedQuota = max(myUsage + residual * myUsageFraction, 1);
+ float calculatedQuota = Math.max(myUsageFraction * confUsage, 1);
val longCalculatedQuota = (long) calculatedQuota;
log.info("computeLocalQuota: myUsage={}, totalUsage={}, myFraction={}; newQuota returned={} [long: {}]",
@@ -93,34 +111,50 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
@Override
// Return true if a report needs to be sent for the current round; false if it can be suppressed for this round.
public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes,
- long currentMessagesUsed, long lastReportedMessages,
- long lastReportTimeMSecsSinceEpoch) {
- // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report.
- long currentTimeMSecs = System.currentTimeMillis();
- long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch;
- if (mSecsSinceLastReport >= ResourceGroupService.maxIntervalForSuppressingReportsMSecs) {
+ long currentMessagesUsed, long lastReportedMessages,
+ long lastReportTimeMSecsSinceEpoch) {
+ ServiceConfiguration configuration = pulsarService.getConfiguration();
+ long resourceUsageMaxUsageReportSuppressRounds = configuration.getResourceUsageMaxUsageReportSuppressRounds();
+ if (resourceUsageMaxUsageReportSuppressRounds == 0) {
return true;
+ } else if (resourceUsageMaxUsageReportSuppressRounds > 0) {
+ // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report.
+ long currentTimeMSecs = System.currentTimeMillis();
+ long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch;
+ if (mSecsSinceLastReport >= TimeUnit.SECONDS.toMillis(
+ (long) configuration.getResourceUsageTransportPublishIntervalInSecs()
+ * configuration.getResourceUsageMaxUsageReportSuppressRounds())) {
+ return true;
+ }
}
// If the percentage change (increase or decrease) in usage is more than a threshold for
// either bytes or messages, send a report.
- final float toleratedDriftPercentage = ResourceGroupService.UsageReportSuppressionTolerancePercentage;
- if (currentBytesUsed > 0) {
- long diff = abs(currentBytesUsed - lastReportedBytes);
- float diffPercentage = (float) diff * 100 / lastReportedBytes;
- if (diffPercentage > toleratedDriftPercentage) {
- return true;
- }
+ final float toleratedDriftPercentage = configuration.getResourceUsageReportSuppressionTolerancePercentage();
+
+ if (needToReportLocalUsage(currentBytesUsed, lastReportedBytes, toleratedDriftPercentage)) {
+ return true;
}
- if (currentMessagesUsed > 0) {
- long diff = abs(currentMessagesUsed - lastReportedMessages);
- float diffPercentage = (float) diff * 100 / lastReportedMessages;
- if (diffPercentage > toleratedDriftPercentage) {
+ if (needToReportLocalUsage(currentMessagesUsed, lastReportedMessages, toleratedDriftPercentage)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private boolean needToReportLocalUsage(long currentUsed, long lastReported, float toleratedDriftPercentage) {
+ if (currentUsed > 0) {
+ if (lastReported == 0) {
return true;
}
+ if (toleratedDriftPercentage <= 0) {
+ return true;
+ }
+ long diff = abs(currentUsed - lastReported);
+ float diffPercentage = (float) diff * 100 / lastReported;
+ return diffPercentage > toleratedDriftPercentage;
}
-
return false;
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 7ca210280a7..2fc8fe883c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -51,7 +51,9 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.resourcegroup.ResourceGroup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.resourcegroup.ResourceGroupPublishLimiter;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService;
import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException;
import org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException;
import org.apache.pulsar.broker.service.BrokerServiceException.ProducerFencedException;
@@ -123,6 +125,9 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
protected volatile ResourceGroupPublishLimiter resourceGroupPublishLimiter;
+ @Getter
+ protected volatile Optional<ResourceGroupDispatchLimiter> resourceGroupDispatchRateLimiter = Optional.empty();
+
protected boolean preciseTopicPublishRateLimitingEnable;
@Getter
@@ -251,6 +256,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
topicPolicies.getSchemaValidationEnforced().updateTopicValue(data.getSchemaValidationEnforced());
topicPolicies.getEntryFilters().updateTopicValue(data.getEntryFilters());
this.subscriptionPolicies = data.getSubscriptionPolicies();
+ topicPolicies.getResourceGroupName().updateTopicValue(data.getResourceGroupName());
updateEntryFilters();
}
@@ -302,6 +308,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
updateNamespaceDispatchRate(namespacePolicies, brokerService.getPulsar().getConfig().getClusterName());
topicPolicies.getSchemaValidationEnforced().updateNamespaceValue(namespacePolicies.schema_validation_enforced);
topicPolicies.getEntryFilters().updateNamespaceValue(namespacePolicies.entryFilters);
+ topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name);
updateEntryFilters();
}
@@ -1175,29 +1182,53 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) {
requireNonNull(namespacePolicies);
- // attach the resource-group level rate limiters, if set
- String rgName = namespacePolicies.resource_group_name;
+ topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name);
+ updateResourceGroupLimiter();
+ }
+
+ public void updateResourceGroupLimiter() {
+ String rgName = topicPolicies.getResourceGroupName().get();
if (rgName != null) {
- final ResourceGroup resourceGroup =
- brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName);
+ ResourceGroupService resourceGroupService = brokerService.getPulsar().getResourceGroupServiceManager();
+ final ResourceGroup resourceGroup = resourceGroupService.resourceGroupGet(rgName);
if (resourceGroup != null) {
+ TopicName topicName = TopicName.get(topic);
+ resourceGroupService.unRegisterTopic(topicName);
+ String topicRg = topicPolicies.getResourceGroupName().getTopicValue();
+ if (topicRg != null) {
+ try {
+ resourceGroupService.registerTopic(topicRg, topicName);
+ } catch (Exception e) {
+ log.error("Failed to register resource group {} for topic {}", rgName, topic);
+ return;
+ }
+ }
this.resourceGroupRateLimitingEnabled = true;
this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter();
this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), this::enableCnxAutoRead);
+ this.resourceGroupDispatchRateLimiter = Optional.of(resourceGroup.getResourceGroupDispatchLimiter());
log.info("Using resource group {} rate limiter for topic {}", rgName, topic);
- return;
}
} else {
- if (this.resourceGroupRateLimitingEnabled) {
- this.resourceGroupPublishLimiter.unregisterRateLimitFunction(this.getName());
- this.resourceGroupPublishLimiter = null;
- this.resourceGroupRateLimitingEnabled = false;
- }
+ closeResourceGroupLimiter();
+
/* Namespace detached from resource group. Enable the producer read */
enableProducerReadForPublishRateLimiting();
}
}
+ protected void closeResourceGroupLimiter() {
+ if (resourceGroupRateLimitingEnabled) {
+ this.resourceGroupPublishLimiter = null;
+ this.resourceGroupDispatchRateLimiter = Optional.empty();
+ this.resourceGroupRateLimitingEnabled = false;
+ }
+ ResourceGroupService resourceGroupServiceManager = brokerService.getPulsar().getResourceGroupServiceManager();
+ if (resourceGroupServiceManager != null) {
+ resourceGroupServiceManager.unRegisterTopic(TopicName.get(topic));
+ }
+ }
+
public void updateEntryFilters() {
if (isSystemTopic()) {
entryFilters = Pair.of(null, Collections.emptyList());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
index 5c314397da8..3f821adda11 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter;
import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
@@ -48,6 +49,10 @@ public interface Replicator {
return Optional.empty();
}
+ default Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+ return Optional.empty();
+ }
+
boolean isConnected();
long getNumberOfEntriesInBacklog();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
index 2e961b793d8..6e7fef67cd9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter;
import org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter;
import org.apache.pulsar.broker.service.plugin.EntryFilter;
@@ -339,6 +340,10 @@ public interface Topic {
return Optional.empty();
}
+ default Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+ return Optional.empty();
+ }
+
default Optional<SubscribeRateLimiter> getSubscribeRateLimiter() {
return Optional.empty();
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index 0a725183bde..250fbbe1e8a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -181,7 +181,7 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema;
}
updatePublishRateLimiter();
- updateResourceGroupLimiter(policies);
+ updateResourceGroupLimiter();
return updateClusterMigrated();
});
}
@@ -457,6 +457,9 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
brokerService.executor().execute(() -> {
brokerService.removeTopicFromCache(NonPersistentTopic.this);
unregisterTopicPolicyListener();
+
+ closeResourceGroupLimiter();
+
log.info("[{}] Topic deleted", topic);
deleteFuture.complete(null);
});
@@ -535,6 +538,7 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
brokerService.executor().execute(() -> {
brokerService.removeTopicFromCache(NonPersistentTopic.this);
unregisterTopicPolicyListener();
+ closeResourceGroupLimiter();
closeFuture.complete(null);
});
}).exceptionally(exception -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
index 3390c3a2885..e5a81fc76f6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
@@ -26,6 +26,7 @@ import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.transaction.TxnID;
@@ -81,8 +82,8 @@ public class GeoPersistentReplicator extends PersistentReplicator {
}
@Override
- protected boolean replicateEntries(List<Entry> entries) {
- boolean atLeastOneMessageSentForReplication = false;
+ protected ReplicationStatus replicateEntries(List<Entry> entries) {
+ ReplicationStatus replicationStatus = ReplicationStatus.NO_ENTRIES_REPLICATED;
boolean isEnableReplicatedSubscriptions =
brokerService.pulsar().getConfiguration().isEnableReplicatedSubscriptions();
@@ -165,7 +166,22 @@ public class GeoPersistentReplicator extends PersistentReplicator {
continue;
}
- dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength()));
+ int msgCount = msg.getMessageBuilder().hasNumMessagesInBatch()
+ ? msg.getMessageBuilder().getNumMessagesInBatch() : 1;
+ dispatchRateLimiter.ifPresent(
+ rateLimiter -> rateLimiter.tryDispatchPermit(msgCount, entry.getLength()));
+
+ ResourceGroupDispatchLimiter resourceGroupLimiter = resourceGroupDispatchRateLimiter.orElse(null);
+ if (resourceGroupLimiter != null) {
+ if (!resourceGroupLimiter.tryAcquire(msgCount, entry.getLength())) {
+ entry.release();
+ msg.recycle();
+ cursor.cancelPendingReadRequest();
+ cursor.rewind();
+ return ReplicationStatus.RATE_LIMITED;
+ }
+ }
+
msg.setReplicatedFrom(localCluster);
headersAndPayload.retain();
@@ -196,15 +212,17 @@ public class GeoPersistentReplicator extends PersistentReplicator {
msg.getMessageBuilder().clearTxnidMostBits();
msg.getMessageBuilder().clearTxnidLeastBits();
msgOut.recordEvent(headersAndPayload.readableBytes());
+ msgOutCounter.add(msgCount);
+ bytesOutCounter.add(length);
// Increment pending messages for messages produced locally
PENDING_MESSAGES_UPDATER.incrementAndGet(this);
producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
- atLeastOneMessageSentForReplication = true;
+ replicationStatus = ReplicationStatus.AT_LEAST_ONE_REPLICATED;
}
}
} catch (Exception e) {
log.error("[{}] Unexpected exception in replication task: {}", replicatorId, e.getMessage(), e);
}
- return atLeastOneMessageSentForReplication;
+ return replicationStatus;
}
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 9e8483be1a7..8b706cfe7d1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -52,6 +52,7 @@ import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory;
import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers;
import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException;
@@ -481,6 +482,27 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul
bytesToRead = calculateToRead.getRight();
}
}
+
+ if (topic.getResourceGroupDispatchRateLimiter().isPresent()) {
+ ResourceGroupDispatchLimiter limiter = topic.getResourceGroupDispatchRateLimiter().get();
+ long availableDispatchRateLimitOnMsg = limiter.getAvailableDispatchRateLimitOnMsg();
+ long availableDispatchRateLimitOnByte = limiter.getAvailableDispatchRateLimitOnByte();
+ if (availableDispatchRateLimitOnMsg == 0 || availableDispatchRateLimitOnByte == 0) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{}, schedule after a {}",
+ name, limiter.getDispatchRateOnMsg(), limiter.getDispatchRateOnByte(),
+ MESSAGE_RATE_BACKOFF_MS);
+ }
+ reScheduleRead();
+ return Pair.of(-1, -1L);
+ } else {
+ Pair<Integer, Long> calculateToRead =
+ computeReadLimits(messagesToRead, (int) availableDispatchRateLimitOnMsg, bytesToRead,
+ availableDispatchRateLimitOnByte);
+ messagesToRead = calculateToRead.getLeft();
+ bytesToRead = calculateToRead.getRight();
+ }
+ }
}
if (havePendingReplayRead) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index a138fde7a72..575be9eaeb5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -40,6 +40,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadE
import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException;
import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer;
import org.apache.pulsar.broker.service.Consumer;
import org.apache.pulsar.broker.service.Dispatcher;
@@ -462,6 +463,30 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher
bytesToRead = calculateToRead.getRight();
}
}
+
+ if (topic.getResourceGroupDispatchRateLimiter().isPresent()) {
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = topic.getResourceGroupDispatchRateLimiter()
+ .get();
+ long availableDispatchRateLimitOnMsg =
+ resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg();
+ long availableDispatchRateLimitOnByte =
+ resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte();
+ if (availableDispatchRateLimitOnMsg == 0 || availableDispatchRateLimitOnByte == 0) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{}, schedule after a {}",
+ name, availableDispatchRateLimitOnMsg, availableDispatchRateLimitOnByte,
+ MESSAGE_RATE_BACKOFF_MS);
+ }
+ reScheduleRead();
+ return Pair.of(-1, -1L);
+ } else {
+ Pair<Integer, Long> calculateToRead =
+ computeReadLimits(messagesToRead, (int) availableDispatchRateLimitOnMsg, bytesToRead,
+ availableDispatchRateLimitOnByte);
+ messagesToRead = calculateToRead.getLeft();
+ bytesToRead = calculateToRead.getRight();
+ }
+ }
}
// If messagesToRead is 0 or less, correct it to 1 to prevent IllegalArgumentException
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
index a0f58e0874f..dd6c6b44cfa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
@@ -33,6 +33,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.LongAdder;
+import lombok.AllArgsConstructor;
+import lombok.Data;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
@@ -48,6 +51,9 @@ import org.apache.bookkeeper.mledger.Position;
import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService;
import org.apache.pulsar.broker.service.AbstractReplicator;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.broker.service.Replicator;
@@ -60,6 +66,7 @@ import org.apache.pulsar.client.impl.ProducerImpl;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.client.impl.SendCallback;
import org.apache.pulsar.common.api.proto.MarkerType;
+import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies;
import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
import org.apache.pulsar.common.schema.SchemaInfo;
import org.apache.pulsar.common.stats.Rate;
@@ -75,6 +82,7 @@ public abstract class PersistentReplicator extends AbstractReplicator
protected final ManagedCursor cursor;
protected Optional<DispatchRateLimiter> dispatchRateLimiter = Optional.empty();
+ protected Optional<ResourceGroupDispatchLimiter> resourceGroupDispatchRateLimiter = Optional.empty();
private final Object dispatchRateLimiterLock = new Object();
private int readBatchSize;
@@ -97,6 +105,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
protected final Rate msgOut = new Rate();
protected final Rate msgExpired = new Rate();
+ protected final LongAdder bytesOutCounter = new LongAdder();
+ protected final LongAdder msgOutCounter = new LongAdder();
protected int messageTTLInSeconds = 0;
@@ -112,8 +122,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
protected volatile boolean fetchSchemaInProgress = false;
public PersistentReplicator(String localCluster, PersistentTopic localTopic, ManagedCursor cursor,
- String remoteCluster, String remoteTopic,
- BrokerService brokerService, PulsarClientImpl replicationClient)
+ String remoteCluster, String remoteTopic,
+ BrokerService brokerService, PulsarClientImpl replicationClient)
throws PulsarServerException {
super(localCluster, localTopic, remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(),
brokerService, replicationClient);
@@ -200,15 +210,32 @@ public abstract class PersistentReplicator extends AbstractReplicator
}
}
+
+ @Data
+ @AllArgsConstructor
+ private static class AvailablePermits {
+ private int messages;
+ private long bytes;
+
+ /**
+ * messages, bytes
+ * 0, O: Producer queue is full, no permits.
+ * -1, -1: Rate Limiter reaches limit.
+ * >0, >0: available permits for read entries.
+ */
+ public boolean isExceeded() {
+ return messages == -1 && bytes == -1;
+ }
+
+ public boolean isReadable() {
+ return messages > 0 && bytes > 0;
+ }
+ }
+
/**
* Calculate available permits for read entries.
- *
- * @return
- * 0: Producer queue is full, no permits.
- * -1: Rate Limiter reaches limit.
- * >0: available permits for read entries.
*/
- private int getAvailablePermits() {
+ private AvailablePermits getAvailablePermits() {
int availablePermits = producerQueueSize - PENDING_MESSAGES_UPDATER.get(this);
// return 0, if Producer queue is full, it will pause read entries.
@@ -217,14 +244,18 @@ public abstract class PersistentReplicator extends AbstractReplicator
log.debug("[{}] Producer queue is full, availablePermits: {}, pause reading",
replicatorId, availablePermits);
}
- return 0;
+ return new AvailablePermits(0, 0);
}
+ long availablePermitsOnMsg = -1;
+ long availablePermitsOnByte = -1;
+
// handle rate limit
if (dispatchRateLimiter.isPresent() && dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) {
DispatchRateLimiter rateLimiter = dispatchRateLimiter.get();
- // no permits from rate limit
- if (!rateLimiter.hasMessageDispatchPermit()) {
+ availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
+ availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte();
+ if (availablePermitsOnByte == 0 || availablePermitsOnMsg == 0) {
if (log.isDebugEnabled()) {
log.debug("[{}] message-read exceeded topic replicator message-rate {}/{},"
+ " schedule after a {}",
@@ -233,17 +264,45 @@ public abstract class PersistentReplicator extends AbstractReplicator
rateLimiter.getDispatchRateOnByte(),
MESSAGE_RATE_BACKOFF_MS);
}
- return -1;
+ return new AvailablePermits(-1, -1);
}
+ }
- // if dispatch-rate is in msg then read only msg according to available permit
- long availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
- if (availablePermitsOnMsg > 0) {
- availablePermits = Math.min(availablePermits, (int) availablePermitsOnMsg);
+ if (resourceGroupDispatchRateLimiter.isPresent()) {
+ ResourceGroupDispatchLimiter rateLimiter = resourceGroupDispatchRateLimiter.get();
+ long rgAvailablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
+ long rgAvailablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte();
+ if (rgAvailablePermitsOnMsg == 0 || rgAvailablePermitsOnByte == 0) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{},"
+ + " schedule after a {}",
+ replicatorId,
+ rateLimiter.getDispatchRateOnMsg(),
+ rateLimiter.getDispatchRateOnByte(),
+ MESSAGE_RATE_BACKOFF_MS);
+ }
+ return new AvailablePermits(-1, -1);
+ }
+ if (availablePermitsOnMsg == -1) {
+ availablePermitsOnMsg = rgAvailablePermitsOnMsg;
+ } else {
+ availablePermitsOnMsg = Math.min(rgAvailablePermitsOnMsg, availablePermitsOnMsg);
+ }
+ if (availablePermitsOnByte == -1) {
+ availablePermitsOnByte = rgAvailablePermitsOnByte;
+ } else {
+ availablePermitsOnByte = Math.min(rgAvailablePermitsOnByte, availablePermitsOnByte);
}
}
- return availablePermits;
+ availablePermitsOnMsg =
+ availablePermitsOnMsg != -1 ? Math.min(availablePermitsOnMsg, availablePermits) : availablePermits;
+ availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize);
+
+ availablePermitsOnByte =
+ availablePermitsOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, availablePermitsOnByte);
+
+ return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte);
}
protected void readMoreEntries() {
@@ -251,10 +310,10 @@ public abstract class PersistentReplicator extends AbstractReplicator
log.info("[{}] Skip the reading due to new detected schema", replicatorId);
return;
}
- int availablePermits = getAvailablePermits();
-
- if (availablePermits > 0) {
- int messagesToRead = Math.min(availablePermits, readBatchSize);
+ AvailablePermits availablePermits = getAvailablePermits();
+ if (availablePermits.isReadable()) {
+ int messagesToRead = availablePermits.getMessages();
+ long bytesToRead = availablePermits.getBytes();
if (!isWritable()) {
if (log.isDebugEnabled()) {
log.debug("[{}] Throttling replication traffic because producer is not writable", replicatorId);
@@ -263,15 +322,13 @@ public abstract class PersistentReplicator extends AbstractReplicator
messagesToRead = 1;
}
- // If messagesToRead is 0 or less, correct it to 1 to prevent IllegalArgumentException
- messagesToRead = Math.max(messagesToRead, 1);
-
// Schedule read
if (HAVE_PENDING_READ_UPDATER.compareAndSet(this, FALSE, TRUE)) {
if (log.isDebugEnabled()) {
- log.debug("[{}] Schedule read of {} messages", replicatorId, messagesToRead);
+ log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, messagesToRead,
+ bytesToRead);
}
- cursor.asyncReadEntriesOrWait(messagesToRead, readMaxSizeBytes, this,
+ cursor.asyncReadEntriesOrWait(messagesToRead, bytesToRead, this,
null, topic.getMaxReadPosition());
} else {
if (log.isDebugEnabled()) {
@@ -279,10 +336,9 @@ public abstract class PersistentReplicator extends AbstractReplicator
replicatorId, messagesToRead);
}
}
- } else if (availablePermits == -1) {
+ } else if (availablePermits.isExceeded()) {
// no permits from rate limit
- topic.getBrokerService().executor().schedule(
- () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS);
+ scheduleReadMoreEntriesWhenExceeded();
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] No Permits for reading. availablePermits: {}",
@@ -310,22 +366,56 @@ public abstract class PersistentReplicator extends AbstractReplicator
readFailureBackoff.reduceToHalf();
- boolean atLeastOneMessageSentForReplication = replicateEntries(entries);
+ ReplicationStatus replicationStatus = replicateEntries(entries);
HAVE_PENDING_READ_UPDATER.set(this, FALSE);
- if (atLeastOneMessageSentForReplication && !isWritable()) {
- // Don't read any more entries until the current pending entries are persisted
- if (log.isDebugEnabled()) {
- log.debug("[{}] Pausing replication traffic. at-least-one: {} is-writable: {}", replicatorId,
- atLeastOneMessageSentForReplication, isWritable());
- }
- } else {
- readMoreEntries();
+ switch (replicationStatus) {
+ case AT_LEAST_ONE_REPLICATED:
+ if (!isWritable()) {
+ // Don't read any more entries until the current pending entries are persisted
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Pausing replication traffic. at-least-one: {} is-writable: {}", replicatorId,
+ true, isWritable());
+ }
+ } else {
+ readMoreEntries();
+ }
+ break;
+ case RATE_LIMITED:
+ scheduleReadMoreEntriesWhenExceeded();
+ break;
+ case NO_ENTRIES_REPLICATED:
+ readMoreEntries();
+ break;
+ default:
+ throw new IllegalStateException("Unexpected replication status: " + replicationStatus);
}
}
- protected abstract boolean replicateEntries(List<Entry> entries);
+ private void scheduleReadMoreEntriesWhenExceeded() {
+ topic.getBrokerService().executor().schedule(
+ () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS);
+ }
+
+ protected enum ReplicationStatus {
+ /**
+ * Replication was blocked due to rate limiting.
+ */
+ RATE_LIMITED,
+
+ /**
+ * At least one entry was successfully replicated.
+ */
+ AT_LEAST_ONE_REPLICATED,
+
+ /**
+ * No entries were replicated.
+ */
+ NO_ENTRIES_REPLICATED
+ }
+
+ protected abstract ReplicationStatus replicateEntries(List<Entry> entries);
protected CompletableFuture<SchemaInfo> getSchemaInfo(MessageImpl msg) throws ExecutionException {
if (msg.getSchemaVersion() == null || msg.getSchemaVersion().length == 0) {
@@ -576,7 +666,7 @@ public abstract class PersistentReplicator extends AbstractReplicator
exception.getMessage(), exception);
if (exception instanceof CursorAlreadyClosedException) {
log.warn("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already"
- + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception);
+ + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception);
// replicator is already deleted and cursor is already closed so, producer should also be disconnected.
terminate();
return;
@@ -605,6 +695,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
stats.replicationBacklog = cursor != null ? cursor.getNumberOfEntriesInBacklog(false) : 0;
stats.connected = producer != null && producer.isConnected();
stats.replicationDelayInSeconds = getReplicationDelayInSeconds();
+ stats.msgOutCounter = msgOutCounter.longValue();
+ stats.bytesOutCounter = bytesOutCounter.longValue();
ProducerImpl producer = this.producer;
if (producer != null) {
@@ -649,6 +741,11 @@ public abstract class PersistentReplicator extends AbstractReplicator
return dispatchRateLimiter;
}
+ @Override
+ public Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+ return resourceGroupDispatchRateLimiter;
+ }
+
@Override
public void initializeDispatchRateLimiterIfNeeded() {
synchronized (dispatchRateLimiterLock) {
@@ -657,6 +754,20 @@ public abstract class PersistentReplicator extends AbstractReplicator
this.dispatchRateLimiter = Optional.of(new DispatchRateLimiter(topic, Type.REPLICATOR));
}
}
+ ResourceGroupService resourceGroupService = brokerService.getPulsar().getResourceGroupServiceManager();
+ HierarchyTopicPolicies hierarchyTopicPolicies = topic.getHierarchyTopicPolicies();
+ String resourceGroupName = hierarchyTopicPolicies.getResourceGroupName().get();
+ if (resourceGroupName != null) {
+ ResourceGroup resourceGroup = resourceGroupService.resourceGroupGet(resourceGroupName);
+ if (resourceGroup != null) {
+ resourceGroupDispatchRateLimiter = Optional.of(resourceGroup
+ .getResourceGroupReplicationDispatchLimiter());
+ }
+ } else {
+ if (resourceGroupDispatchRateLimiter.isPresent()) {
+ resourceGroupDispatchRateLimiter = Optional.empty();
+ }
+ }
}
@Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 7faa86e5705..aa3bfa45087 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -426,7 +426,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
if (!optPolicies.isPresent()) {
isEncryptionRequired = false;
updatePublishRateLimiter();
- updateResourceGroupLimiter(new Policies());
+ updateResourceGroupLimiter();
initializeDispatchRateLimiterIfNeeded();
updateSubscribeRateLimiter();
return;
@@ -442,7 +442,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
updatePublishRateLimiter();
- updateResourceGroupLimiter(policies);
+ updateResourceGroupLimiter();
this.isEncryptionRequired = policies.encryption_required;
@@ -1493,6 +1493,8 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
unregisterTopicPolicyListener();
+ closeResourceGroupLimiter();
+
log.info("[{}] Topic deleted", topic);
deleteFuture.complete(null);
}
@@ -1703,6 +1705,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close);
unregisterTopicPolicyListener();
+
+ closeResourceGroupLimiter();
+
log.info("[{}] Topic closed", topic);
cancelFencedTopicMonitoringTask();
closeFuture.complete(null);
@@ -3078,7 +3083,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
// Apply policies for components.
List<CompletableFuture<Void>> applyPolicyTasks = applyUpdatedTopicPolicies();
- applyPolicyTasks.add(applyUpdatedNamespacePolicies(data));
+ applyPolicyTasks.add(applyUpdatedNamespacePolicies());
return FutureUtil.waitForAll(applyPolicyTasks)
.thenAccept(__ -> log.info("[{}] namespace-level policies updated successfully", topic))
.exceptionally(ex -> {
@@ -3087,8 +3092,8 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
});
}
- private CompletableFuture<Void> applyUpdatedNamespacePolicies(Policies namespaceLevelPolicies) {
- return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter(namespaceLevelPolicies));
+ private CompletableFuture<Void> applyUpdatedNamespacePolicies() {
+ return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter());
}
private List<CompletableFuture<Void>> applyUpdatedTopicPolicies() {
@@ -3107,6 +3112,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateDispatchRateLimiter()));
applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscribeRateLimiter()));
applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updatePublishRateLimiter()));
+ applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter()));
applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscriptionsDispatcherRateLimiter()));
applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
index b48f748bf5d..9fde570a042 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
@@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.client.impl.MessageIdImpl;
import org.apache.pulsar.client.impl.MessageImpl;
@@ -54,8 +55,8 @@ public class ShadowReplicator extends PersistentReplicator {
}
@Override
- protected boolean replicateEntries(List<Entry> entries) {
- boolean atLeastOneMessageSentForReplication = false;
+ protected ReplicationStatus replicateEntries(List<Entry> entries) {
+ ReplicationStatus replicationStatus = ReplicationStatus.NO_ENTRIES_REPLICATED;
try {
// This flag is set to true when we skip at least one local message,
@@ -89,9 +90,26 @@ public class ShadowReplicator extends PersistentReplicator {
continue;
}
- dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength()));
+
+ int msgCount = msg.getMessageBuilder().hasNumMessagesInBatch()
+ ? msg.getMessageBuilder().getNumMessagesInBatch() : 1;
+ dispatchRateLimiter.ifPresent(
+ rateLimiter -> rateLimiter.tryDispatchPermit(msgCount, entry.getLength()));
+
+ ResourceGroupDispatchLimiter resourceGroupLimiter = resourceGroupDispatchRateLimiter.orElse(null);
+ if (resourceGroupLimiter != null) {
+ if (!resourceGroupLimiter.tryAcquire(msgCount, entry.getLength())) {
+ entry.release();
+ msg.recycle();
+ cursor.cancelPendingReadRequest();
+ cursor.rewind();
+ return ReplicationStatus.RATE_LIMITED;
+ }
+ }
msgOut.recordEvent(headersAndPayload.readableBytes());
+ msgOutCounter.add(msgCount);
+ bytesOutCounter.add(length);
msg.setReplicatedFrom(localCluster);
@@ -102,13 +120,13 @@ public class ShadowReplicator extends PersistentReplicator {
// Increment pending messages for messages produced locally
PENDING_MESSAGES_UPDATER.incrementAndGet(this);
producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
- atLeastOneMessageSentForReplication = true;
+ replicationStatus = ReplicationStatus.AT_LEAST_ONE_REPLICATED;
}
} catch (Exception e) {
log.error("[{}] Unexpected exception in replication task for shadow topic: {}",
replicatorId, e.getMessage(), e);
}
- return atLeastOneMessageSentForReplication;
+ return replicationStatus;
}
/**
diff --git a/pulsar-broker/src/main/proto/ResourceUsage.proto b/pulsar-broker/src/main/proto/ResourceUsage.proto
index 4706c9dfbcd..ac7e1dea234 100644
--- a/pulsar-broker/src/main/proto/ResourceUsage.proto
+++ b/pulsar-broker/src/main/proto/ResourceUsage.proto
@@ -38,6 +38,7 @@ message ResourceUsage {
optional NetworkUsage publish = 2;
optional NetworkUsage dispatch = 3;
optional StorageUsage storage = 4;
+ optional NetworkUsage replicationDispatch = 6;
}
message ResourceUsageInfo {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java
new file mode 100644
index 00000000000..4222862176b
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.broker.service.BrokerTestBase;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ResourceGroup;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AdminResource}.
+ */
+@Test(groups = "broker-admin")
+public class AdminResourceGroupTest extends BrokerTestBase {
+
+ @BeforeClass
+ @Override
+ public void setup() throws Exception {
+ super.baseSetup();
+ }
+
+ @AfterClass(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Override
+ protected void doInitConf() throws Exception {
+ super.doInitConf();
+ conf.setTopicLevelPoliciesEnabled(true);
+ conf.setSystemTopicEnabled(true);
+ }
+
+ @Test
+ public void testTopicResourceGroup() throws PulsarAdminException {
+ String topic = newTopicName();
+ TopicName topicName = TopicName.get(topic);
+
+ String resourceGroupName = "rg-topic-" + UUID.randomUUID();
+ ResourceGroup resourceGroup = new ResourceGroup();
+ resourceGroup.setPublishRateInMsgs(1000);
+ resourceGroup.setPublishRateInBytes(100000L);
+ resourceGroup.setDispatchRateInMsgs(2000);
+ resourceGroup.setDispatchRateInBytes(200000L);
+ admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+
+ admin.topics().createNonPartitionedTopic(topic);
+
+ admin.topicPolicies().setResourceGroup(topic, resourceGroupName);
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(admin.topicPolicies().getResourceGroup(topic, true), resourceGroupName);
+ });
+
+ Awaitility.await().untilAsserted(() -> {
+ org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+ .getTopicResourceGroup(topicName);
+ assertNotNull(rg);
+ assertEquals(rg.resourceGroupName, resourceGroupName);
+ });
+
+ admin.topicPolicies().removeResourceGroup(topic);
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(StringUtils.isEmpty(admin.topicPolicies().getResourceGroup(topic, true)));
+ org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+ .getTopicResourceGroup(topicName);
+ assertNull(rg);
+ });
+ }
+
+ @Test
+ public void testTopicResourceGroupOverriderNamespaceResourceGroup() throws PulsarAdminException {
+ String namespaceResourceGroupName = "rg-ns-" + UUID.randomUUID();
+ ResourceGroup namespaceResourceGroup = new ResourceGroup();
+ namespaceResourceGroup.setPublishRateInMsgs(1001);
+ namespaceResourceGroup.setPublishRateInBytes(100001L);
+ namespaceResourceGroup.setDispatchRateInMsgs(2001);
+ namespaceResourceGroup.setDispatchRateInBytes(200001L);
+ admin.resourcegroups().createResourceGroup(namespaceResourceGroupName, namespaceResourceGroup);
+
+ String topicResourceGroupName = "rg-topic-" + UUID.randomUUID();
+ ResourceGroup topicResourceGroup = new ResourceGroup();
+ topicResourceGroup.setPublishRateInMsgs(1000);
+ topicResourceGroup.setPublishRateInBytes(100000L);
+ topicResourceGroup.setDispatchRateInMsgs(2000);
+ topicResourceGroup.setDispatchRateInBytes(200000L);
+ admin.resourcegroups().createResourceGroup(topicResourceGroupName, topicResourceGroup);
+
+ String topic = newTopicName();
+ TopicName topicName = TopicName.get(topic);
+ String namespace = topicName.getNamespace();
+ admin.namespaces().setNamespaceResourceGroup(namespace, namespaceResourceGroupName);
+ assertEquals(admin.namespaces().getNamespaceResourceGroup(namespace), namespaceResourceGroupName);
+
+ admin.topics().createNonPartitionedTopic(topic);
+ admin.topicPolicies().setResourceGroup(topic, topicResourceGroupName);
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(admin.topicPolicies()
+ .getResourceGroup(topic, true), topicResourceGroupName);
+ org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+ .getTopicResourceGroup(topicName);
+ assertNotNull(rg);
+ assertEquals(rg.resourceGroupName, topicResourceGroupName);
+ });
+
+ admin.topicPolicies().removeResourceGroup(topic);
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(admin.topicPolicies()
+ .getResourceGroup(topic, true), namespaceResourceGroupName);
+ org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+ .getTopicResourceGroup(topicName);
+ assertNull(rg);
+ });
+ }
+
+ @Test
+ public void testUpdateResourceGroup() throws PulsarAdminException {
+ String resourceGroupName = "rg-" + UUID.randomUUID();
+ ResourceGroup resourceGroup = new ResourceGroup();
+ resourceGroup.setPublishRateInMsgs(1000);
+ resourceGroup.setPublishRateInBytes(100000L);
+ resourceGroup.setDispatchRateInMsgs(2000);
+ resourceGroup.setDispatchRateInBytes(200000L);
+ resourceGroup.setReplicationDispatchRateInMsgs(10L);
+ resourceGroup.setReplicationDispatchRateInBytes(20L);
+
+ admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+ ResourceGroup got = admin.resourcegroups().getResourceGroup(resourceGroupName);
+ assertEquals(got, resourceGroup);
+
+ resourceGroup.setReplicationDispatchRateInMsgs(11L);
+ resourceGroup.setReplicationDispatchRateInBytes(29L);
+ admin.resourcegroups().updateResourceGroup(resourceGroupName, resourceGroup);
+ got = admin.resourcegroups().getResourceGroup(resourceGroupName);
+ assertEquals(got, resourceGroup);
+
+ admin.resourcegroups().deleteResourceGroup(resourceGroupName);
+ assertThrows(PulsarAdminException.NotFoundException.class,
+ () -> admin.resourcegroups().getResourceGroup(resourceGroupName));
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java
new file mode 100644
index 00000000000..e0142a74b3c
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertEquals;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
+import org.testng.annotations.Test;
+
+public class ResourceGroupMetricTest {
+ @Test
+ public void testLocalQuotaMetric() {
+ String rgName = "my-resource-group";
+ ResourceGroupMonitoringClass publish = ResourceGroupMonitoringClass.Publish;
+ int reportPeriod = 2;
+ BytesAndMessagesCount b = new BytesAndMessagesCount();
+ b.messages = 10;
+ b.bytes = 20;
+ int incTimes = 2;
+ for (int i = 0; i < 2; i++) {
+ ResourceGroupService.incRgCalculatedQuota(rgName, publish, b, reportPeriod);
+ }
+ double rgLocalUsageByteCount = ResourceGroupService.getRgQuotaByteCount(rgName, publish.name());
+ double rgQuotaMessageCount = ResourceGroupService.getRgQuotaMessageCount(rgName, publish.name());
+ assertEquals(rgLocalUsageByteCount, incTimes * b.bytes * reportPeriod);
+ assertEquals(rgQuotaMessageCount, incTimes * b.messages * reportPeriod);
+
+ double rgLocalUsageByte = ResourceGroupService.getRgQuotaByte(rgName, publish.name());
+ double rgQuotaMessage = ResourceGroupService.getRgQuotaMessage(rgName, publish.name());
+ assertEquals(rgLocalUsageByte, b.bytes);
+ assertEquals(rgQuotaMessage, b.messages);
+ }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java
new file mode 100644
index 00000000000..fc9862456d4
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.testng.annotations.Test;
+
+public class ResourceGroupRateLimiterManagerTest {
+
+ @Test
+ public void testNewReplicationDispatchRateLimiterWithEmptyResourceGroup() {
+ org.apache.pulsar.common.policies.data.ResourceGroup emptyResourceGroup =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+
+ @Cleanup(value = "shutdown")
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ @Cleanup
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(emptyResourceGroup,executorService);
+ assertFalse(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L);
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L);
+ }
+
+ @Test
+ public void testReplicationDispatchRateLimiterOnMsgs() {
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInMsgs(10L);
+ @Cleanup(value = "shutdown")
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ @Cleanup
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+ assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L);
+ }
+
+ @Test
+ public void testReplicationDispatchRateLimiterOnBytes() {
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInBytes(20L);
+ @Cleanup(value = "shutdown")
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ @Cleanup
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+ assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L);
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+ }
+
+ @Test
+ public void testUpdateReplicationDispatchRateLimiter() {
+ org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInMsgs(10L);
+ resourceGroup.setReplicationDispatchRateInBytes(100L);
+
+ @Cleanup(value = "shutdown")
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+ @Cleanup
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+
+ BytesAndMessagesCount quota = new BytesAndMessagesCount();
+ quota.messages = 20;
+ quota.bytes = 200;
+ ResourceGroupRateLimiterManager.updateReplicationDispatchRateLimiter(resourceGroupDispatchLimiter, quota);
+
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), quota.bytes);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), quota.bytes);
+ assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), quota.messages);
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), quota.messages);
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
index f47f7686743..0a9a071252f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
@@ -18,25 +18,31 @@
*/
package org.apache.pulsar.broker.resourcegroup;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Policy.Expiration;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
-import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields;
import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerBrokerUsageStats;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
import org.apache.pulsar.broker.service.resource.usage.NetworkUsage;
import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.ClusterData;
+import org.awaitility.Awaitility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
-import org.testng.Assert;
import org.testng.annotations.Test;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
@BeforeClass
@@ -51,7 +57,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
long currentMessagesUsed, long lastReportedMessages,
long lastReportTimeMSecsSinceEpoch)
{
- final int maxSuppressRounds = ResourceGroupService.MaxUsageReportSuppressRounds;
+ final int maxSuppressRounds = conf.getResourceUsageMaxUsageReportSuppressRounds();
if (++numLocalReportsEvaluated % maxSuppressRounds == (maxSuppressRounds - 1)) {
return true;
}
@@ -87,7 +93,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
@Test
public void measureOpsTime() throws PulsarAdminException {
long mSecsStart, mSecsEnd, diffMsecs;
- final int numPerfTestIterations = 10_000_000;
+ final int numPerfTestIterations = 1_000_000;
org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
new org.apache.pulsar.common.policies.data.ResourceGroup();
BytesAndMessagesCount stats = new BytesAndMessagesCount();
@@ -111,16 +117,18 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs)/numPerfTestIterations);
// Going through the resource-group service
- final String tenantName = "SomeTenant";
- final String namespaceName = "SomeNameSpace";
+ final TopicName topicName = TopicName.get("SomeTenant/SomeNameSpace/my-topic");
+ rgs.registerTopic(rgName, topicName);
+ final String tenantName = topicName.getTenant();
+ final String namespaceName = topicName.getNamespace();
rgs.registerTenant(rgName, tenantName);
- final NamespaceName tenantAndNamespaceName = NamespaceName.get(tenantName, namespaceName);
+ final NamespaceName tenantAndNamespaceName = topicName.getNamespaceObject();
rgs.registerNameSpace(rgName, tenantAndNamespaceName);
mSecsStart = System.currentTimeMillis();
for (int ix = 0; ix < numPerfTestIterations; ix++) {
for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) {
monClass = ResourceGroupMonitoringClass.values()[monClassIdx];
- rgs.incrementUsage(tenantName, namespaceName, monClass, stats);
+ rgs.incrementUsage(tenantName, namespaceName, topicName.toString(), monClass, stats);
}
}
mSecsEnd = System.currentTimeMillis();
@@ -129,6 +137,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs)/numPerfTestIterations);
rgs.unRegisterTenant(rgName, tenantName);
rgs.unRegisterNameSpace(rgName, tenantAndNamespaceName);
+ rgs.unRegisterTopic(topicName);
// The overhead of a RG lookup
mSecsStart = System.currentTimeMillis();
@@ -153,6 +162,8 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
rgConfig.setPublishRateInMsgs(100);
rgConfig.setDispatchRateInBytes(40000L);
rgConfig.setDispatchRateInMsgs(500);
+ rgConfig.setReplicationDispatchRateInBytes(2000L);
+ rgConfig.setReplicationDispatchRateInMsgs(400L);
int initialNumQuotaCalculations = numAnonymousQuotaCalculations;
rgs.resourceGroupCreate(rgName, rgConfig);
@@ -167,6 +178,8 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs()*10);
rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes()/10);
rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs()/10);
+ rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes()/10);
+ rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs()/10);
rgs.resourceGroupUpdate(rgName, rgConfig);
Assert.assertEquals(rgs.getNumResourceGroups(), 1);
@@ -184,6 +197,9 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.Dispatch.ordinal()];
Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getDispatchRateInBytes().longValue());
Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getDispatchRateInMsgs().intValue());
+ monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.ReplicationDispatch.ordinal()];
+ Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getReplicationDispatchRateInBytes().longValue());
+ Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getReplicationDispatchRateInMsgs().intValue());
Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(randomRgName));
@@ -197,6 +213,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
final NamespaceName tenantAndNamespace = NamespaceName.get(tenantName, namespaceName);
rgs.registerNameSpace(rgName, tenantAndNamespace);
+ rgs.registerTopic(rgName, topic);
// Delete of our valid config should throw until we unref correspondingly.
Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(rgName));
@@ -213,6 +230,10 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
// Gross hack!
if (monClass == ResourceGroupMonitoringClass.Publish) {
nwUsage = usage.setPublish();
+ } else if (monClass == ResourceGroupMonitoringClass.Dispatch) {
+ nwUsage = usage.setDispatch();
+ } else if (monClass == ResourceGroupMonitoringClass.ReplicationDispatch) {
+ nwUsage = usage.setReplicationDispatch();
} else {
nwUsage = usage.setDispatch();
}
@@ -220,7 +241,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
// maxUsageReportSuppressRounds iterations. So, if we run for maxUsageReportSuppressRounds iterations,
// we should see needToReportLocalUsage() return true at least once.
Set<Boolean> myBoolSet = new HashSet<>();
- for (int idx = 0; idx < ResourceGroupService.MaxUsageReportSuppressRounds; idx++) {
+ for (int idx = 0; idx < conf.getResourceUsageMaxUsageReportSuppressRounds(); idx++) {
needToReport = retRG.setUsageInMonitoredEntity(monClass, nwUsage);
myBoolSet.add(needToReport);
}
@@ -231,6 +252,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
rgs.unRegisterTenant(rgName, tenantName);
rgs.unRegisterNameSpace(rgName, tenantAndNamespace);
+ rgs.unRegisterTopic(topic);
BytesAndMessagesCount publishQuota = rgs.getPublishRateLimiters(rgName);
@@ -255,6 +277,101 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
Assert.assertThrows(PulsarAdminException.class, () -> rgs.getPublishRateLimiters(rgName));
Assert.assertEquals(rgs.getNumResourceGroups(), 0);
+
+ Assert.assertEquals(rgs.getTopicConsumeStats().estimatedSize(), 0);
+ Assert.assertEquals(rgs.getTopicProduceStats().estimatedSize(), 0);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().estimatedSize(), 0);
+ Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+ }
+
+ @Test
+ public void testCleanupStatsWhenUnRegisterTopic()
+ throws PulsarAdminException {
+ String tenantName = UUID.randomUUID().toString();
+ org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ final String rgName = UUID.randomUUID().toString();
+ rgConfig.setPublishRateInBytes(15000L);
+ rgConfig.setPublishRateInMsgs(100);
+ rgConfig.setDispatchRateInBytes(40000L);
+ rgConfig.setDispatchRateInMsgs(500);
+ rgConfig.setReplicationDispatchRateInBytes(2000L);
+ rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+ rgs.resourceGroupCreate(rgName, rgConfig);
+ String nsName = tenantName + "/" + UUID.randomUUID();
+ TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID());
+ String topic = topicName.toString();
+
+ rgs.registerTopic(rgName, topicName);
+
+ // Simulate replicator
+ rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.ReplicationDispatch);
+ rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.Publish);
+ rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.Dispatch);
+ Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1);
+ Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1);
+ Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 1);
+ Set<String> replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement());
+ Assert.assertEquals(replicators.size(), 1);
+
+ rgs.unRegisterTopic(TopicName.get(topic));
+
+ Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+
+ rgs.resourceGroupDelete(rgName);
+ }
+
+ @Test
+ public void testCleanupStatsWhenUnRegisterNamespace()
+ throws PulsarAdminException {
+ String tenantName = UUID.randomUUID().toString();
+ org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ final String rgName = UUID.randomUUID().toString();
+ rgConfig.setPublishRateInBytes(15000L);
+ rgConfig.setPublishRateInMsgs(100);
+ rgConfig.setDispatchRateInBytes(40000L);
+ rgConfig.setDispatchRateInMsgs(500);
+ rgConfig.setReplicationDispatchRateInBytes(2000L);
+ rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+ rgs.resourceGroupCreate(rgName, rgConfig);
+ String nsName = tenantName + "/" + UUID.randomUUID();
+ TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID());
+ String topic = topicName.toString();
+
+ rgs.registerNameSpace(rgName, topicName.getNamespaceObject());
+
+ // Simulate replicator
+ rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.ReplicationDispatch);
+ rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.Publish);
+ rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+ ResourceGroupMonitoringClass.Dispatch);
+ Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1);
+ Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1);
+ Set<String> replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement());
+ Assert.assertEquals(replicators.size(), 1);
+
+ rgs.unRegisterNameSpace(rgName, topicName.getNamespaceObject());
+
+ Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+ Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+
+ rgs.resourceGroupDelete(rgName);
}
@Test
@@ -265,6 +382,75 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
Assert.assertTrue(service.getCalculateQuotaPeriodicTask().isCancelled());
}
+ private void assertTopicStatsCache(Cache<String, BytesAndMessagesCount> cache, long durationMS) {
+ Optional<Expiration<String, BytesAndMessagesCount>> expirationOptional =
+ cache.policy().expireAfterAccess();
+ Assert.assertTrue(expirationOptional.isPresent());
+ Expiration<String, BytesAndMessagesCount> expiration = expirationOptional.get();
+ Assert.assertEquals(expiration.getExpiresAfter().toMillis(), durationMS);
+ }
+
+ @Test
+ public void testTopicStatsCache() {
+ long ms = 2_000;
+ Cache<String, BytesAndMessagesCount> cache =
+ pulsar.getResourceGroupServiceManager().newStatsCache(TimeUnit.MILLISECONDS.toMillis(ms));
+ String key = "topic-1";
+ BytesAndMessagesCount value = new BytesAndMessagesCount();
+ cache.put(key, value);
+ Assert.assertEquals(cache.getIfPresent(key), value);
+ Awaitility.await().pollDelay(ms + 200 , TimeUnit.MILLISECONDS).untilAsserted(() -> {
+ Assert.assertNull(cache.getIfPresent(key));
+ });
+
+ long expMS =
+ TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs())
+ * 2;
+ assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicConsumeStats(), expMS);
+ assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicProduceStats(), expMS);
+ assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getReplicationDispatchStats(), expMS);
+ }
+
+ @Test
+ public void testBrokerStatsCache() throws PulsarAdminException {
+ long ms = 2_000;
+ String key = "broker-1";
+ PerBrokerUsageStats value = new PerBrokerUsageStats();
+ PerMonitoringClassFields perMonitoringClassFields = PerMonitoringClassFields.create(ms);
+ Cache<String, PerBrokerUsageStats> usageFromOtherBrokers = perMonitoringClassFields.usageFromOtherBrokers;
+ usageFromOtherBrokers.put(key, value);
+ Assert.assertEquals(usageFromOtherBrokers.getIfPresent(key), value);
+ Awaitility.await().atMost(ms + 500, TimeUnit.MILLISECONDS).untilAsserted(() -> {
+ Assert.assertNull(usageFromOtherBrokers.getIfPresent(key));
+ });
+
+ org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ final String rgName = UUID.randomUUID().toString();
+ rgConfig.setPublishRateInBytes(15000L);
+ rgConfig.setPublishRateInMsgs(100);
+ rgConfig.setDispatchRateInBytes(40000L);
+ rgConfig.setDispatchRateInMsgs(500);
+ rgConfig.setReplicationDispatchRateInBytes(2000L);
+ rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+ ResourceGroupService resourceGroupServiceManager = pulsar.getResourceGroupServiceManager();
+ resourceGroupServiceManager.resourceGroupCreate(rgName, rgConfig);
+ ResourceGroup resourceGroup = resourceGroupServiceManager.resourceGroupGet(rgName);
+ PerMonitoringClassFields publishMonitoredEntity =
+ resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Publish);
+ Cache<String, PerBrokerUsageStats> cache = publishMonitoredEntity.usageFromOtherBrokers;
+ Optional<Expiration<String, PerBrokerUsageStats>> expirationOptional =
+ cache.policy().expireAfterWrite();
+ Assert.assertTrue(expirationOptional.isPresent());
+ Expiration<String, PerBrokerUsageStats> brokerUsageStatsExpiration = expirationOptional.get();
+
+ long statsDuration =
+ TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs())
+ * conf.getResourceUsageMaxUsageReportSuppressRounds() * 2;
+ Assert.assertEquals(brokerUsageStatsExpiration.getExpiresAfter().toMillis(), statsDuration);
+ }
+
private ResourceGroupService rgs;
int numAnonymousQuotaCalculations;
@@ -272,7 +458,6 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
private static final int PUBLISH_INTERVAL_SECS = 500;
private void prepareData() throws PulsarAdminException {
- this.conf.setResourceUsageTransportPublishIntervalInSecs(PUBLISH_INTERVAL_SECS);
admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
}
}
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java
new file mode 100644
index 00000000000..fa87b2ef705
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertNotNull;
+import com.google.common.collect.Sets;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupUsageStatsType;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.SubscriptionInitialPosition;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class ResourceGroupUsageAggregationOnTopicLevelTest extends ProducerConsumerBase {
+
+ private final String TenantName = "pulsar-test";
+ private final String NsName = "test";
+ private final String TenantAndNsName = TenantName + "/" + NsName;
+ private final String TestProduceConsumeTopicName = "/test/prod-cons-topic";
+ private final String PRODUCE_CONSUME_PERSISTENT_TOPIC = "persistent://" + TenantAndNsName + TestProduceConsumeTopicName;
+ private final String PRODUCE_CONSUME_NON_PERSISTENT_TOPIC =
+ "non-persistent://" + TenantAndNsName + TestProduceConsumeTopicName;
+
+ @BeforeClass(alwaysRun = true)
+ @Override
+ protected void setup() throws Exception {
+ super.internalSetup();
+ this.conf.setAllowAutoTopicCreation(true);
+
+ final String clusterName = "test";
+ admin.clusters().createCluster(clusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build());
+ admin.tenants().createTenant(TenantName,
+ new TenantInfoImpl(Sets.newHashSet("fakeAdminRole"), Sets.newHashSet(clusterName)));
+ admin.namespaces().createNamespace(TenantAndNsName);
+ admin.namespaces().setNamespaceReplicationClusters(TenantAndNsName, Sets.newHashSet(clusterName));
+ }
+
+ @AfterClass(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test
+ public void testPersistentTopicProduceConsumeUsageOnRG() throws Exception {
+ testProduceConsumeUsageOnRG(PRODUCE_CONSUME_PERSISTENT_TOPIC);
+ }
+
+ @Test
+ public void testNonPersistentTopicProduceConsumeUsageOnRG() throws Exception {
+ testProduceConsumeUsageOnRG(PRODUCE_CONSUME_NON_PERSISTENT_TOPIC);
+ }
+
+ private void testProduceConsumeUsageOnRG(String topicString) throws Exception {
+ ResourceQuotaCalculator dummyQuotaCalc = new ResourceQuotaCalculator() {
+ @Override
+ public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes,
+ long currentMessagesUsed, long lastReportedMessages,
+ long lastReportTimeMSecsSinceEpoch) {
+ return false;
+ }
+
+ @Override
+ public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) {
+ return 0;
+ }
+ };
+
+ @Cleanup
+ ResourceUsageTopicTransportManager transportMgr = new ResourceUsageTopicTransportManager(pulsar);
+ @Cleanup
+ ResourceGroupService rgs = new ResourceGroupService(pulsar, TimeUnit.MILLISECONDS, transportMgr,
+ dummyQuotaCalc);
+
+ String activeRgName = "runProduceConsume";
+ ResourceGroup activeRG;
+
+ ResourceUsagePublisher ruP = new ResourceUsagePublisher() {
+ @Override
+ public String getID() {
+ return rgs.resourceGroupGet(activeRgName).resourceGroupName;
+ }
+
+ @Override
+ public void fillResourceUsage(ResourceUsage resourceUsage) {
+ rgs.resourceGroupGet(activeRgName).rgFillResourceUsage(resourceUsage);
+ }
+ };
+
+ ResourceUsageConsumer ruC = new ResourceUsageConsumer() {
+ @Override
+ public String getID() {
+ return rgs.resourceGroupGet(activeRgName).resourceGroupName;
+ }
+
+ @Override
+ public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) {
+ rgs.resourceGroupGet(activeRgName).rgResourceUsageListener(broker, resourceUsage);
+ }
+ };
+
+ org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+ new org.apache.pulsar.common.policies.data.ResourceGroup();
+ rgConfig.setPublishRateInBytes(1500L);
+ rgConfig.setPublishRateInMsgs(100);
+ rgConfig.setDispatchRateInBytes(4000L);
+ rgConfig.setPublishRateInMsgs(500);
+
+ rgs.resourceGroupCreate(activeRgName, rgConfig, ruP, ruC);
+
+ activeRG = rgs.resourceGroupGet(activeRgName);
+ assertNotNull(activeRG);
+
+ String subscriptionName = "my-subscription";
+ @Cleanup
+ Consumer<byte[]> consumer = pulsarClient.newConsumer()
+ .topic(topicString)
+ .subscriptionName(subscriptionName)
+ .subscriptionType(SubscriptionType.Shared)
+ .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+ .subscribe();
+
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer()
+ .topic(topicString)
+ .create();
+
+ TopicName myTopic = TopicName.get(topicString);
+ rgs.unRegisterTopic(myTopic);
+ rgs.registerTopic(activeRgName,myTopic);
+
+ final int NumMessagesToSend = 10;
+ int sentNumBytes = 0;
+ int sentNumMsgs = 0;
+ for (int ix = 0; ix < NumMessagesToSend; ix++) {
+ byte[] mesg = String.format("Hi, ix=%s", ix).getBytes();
+ producer.send(mesg);
+ sentNumBytes += mesg.length;
+ sentNumMsgs++;
+ }
+
+ this.verifyStats(rgs, topicString, activeRgName, sentNumBytes, sentNumMsgs, 0, 0,
+ true, false);
+
+ int recvdNumBytes = 0;
+ int recvdNumMsgs = 0;
+
+ Message<byte[]> message;
+ while (recvdNumMsgs < sentNumMsgs) {
+ message = consumer.receive();
+ recvdNumBytes += message.getValue().length;
+ recvdNumMsgs++;
+ }
+
+ this.verifyStats(rgs,topicString, activeRgName, sentNumBytes, sentNumMsgs, recvdNumBytes, recvdNumMsgs,
+ true, true);
+ }
+
+ private void verifyStats(ResourceGroupService rgs, String topicString, String rgName,
+ int sentNumBytes, int sentNumMsgs,
+ int recvdNumBytes, int recvdNumMsgs,
+ boolean checkProduce, boolean checkConsume) throws PulsarAdminException {
+ BrokerService bs = pulsar.getBrokerService();
+ Awaitility.await().untilAsserted(() -> {
+ TopicStatsImpl topicStats = bs.getTopicStats().get(topicString);
+ assertNotNull(topicStats);
+ if (checkProduce) {
+ Assert.assertTrue(topicStats.bytesInCounter >= sentNumBytes);
+ Assert.assertEquals(sentNumMsgs, topicStats.msgInCounter);
+ }
+ if (checkConsume) {
+ Assert.assertTrue(topicStats.bytesOutCounter >= recvdNumBytes);
+ Assert.assertEquals(recvdNumMsgs, topicStats.msgOutCounter);
+ }
+ });
+ if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
+ rgs.aggregateResourceGroupLocalUsages();
+ BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish,
+ ResourceGroupUsageStatsType.Cumulative);
+ BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch,
+ ResourceGroupUsageStatsType.Cumulative);
+
+ if (checkProduce) {
+ Assert.assertTrue(prodCounts.bytes >= sentNumBytes);
+ Assert.assertEquals(sentNumMsgs, prodCounts.messages);
+ }
+ if (checkConsume) {
+ Assert.assertTrue(consCounts.bytes >= recvdNumBytes);
+ Assert.assertEquals(recvdNumMsgs, consCounts.messages);
+ }
+ }
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
index 72fd91ea46d..84c4926b5f9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
@@ -191,8 +191,6 @@ public class ResourceGroupUsageAggregationTest extends ProducerConsumerBase {
}
}
}
- rgs.getTopicConsumeStats().clear();
- rgs.getTopicProduceStats().clear();
rgs.unRegisterTenant(activeRgName, tenantString);
rgs.unRegisterNameSpace(activeRgName, NamespaceName.get(nsString));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
index eeda957bf3e..73033f36530 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
@@ -32,7 +32,7 @@ public class ResourceQuotaCalculatorImplTest extends MockedPulsarServiceBaseTest
@Override
protected void setup() throws Exception {
super.internalSetup();
- this.rqCalc = new ResourceQuotaCalculatorImpl();
+ this.rqCalc = new ResourceQuotaCalculatorImpl(pulsar);
}
@AfterClass(alwaysRun = true)
@@ -121,5 +121,16 @@ public class ResourceQuotaCalculatorImplTest extends MockedPulsarServiceBaseTest
Assert.assertTrue(rqCalc.needToReportLocalUsage(940, 1000, 94, 100, System.currentTimeMillis()));
}
+ @Test
+ public void testUsedUsageLessThanConfigUsage() throws PulsarAdminException {
+ final long config = 100;
+ final long usedUsage = 100 / pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs() / 3;
+ final long[] allUsage = {usedUsage, usedUsage};
+ for (long n : allUsage) {
+ long localQuota = rqCalc.computeLocalQuota(config, n, allUsage);
+ Assert.assertEquals(localQuota, config);
+ }
+ }
+
private ResourceQuotaCalculatorImpl rqCalc;
}
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
index 8cf5511a684..9eacb76f371 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
@@ -80,7 +80,7 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
resourceUsage.setOwner(getID());
resourceUsage.setPublish().setMessagesPerPeriod(1000).setBytesPerPeriod(10001);
resourceUsage.setStorage().setTotalBytes(500003);
-
+ resourceUsage.setReplicationDispatch().setMessagesPerPeriod(2000).setBytesPerPeriod(4000);
}
};
@@ -99,6 +99,10 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
p.setBytesPerPeriod(resourceUsage.getPublish().getBytesPerPeriod());
p.setMessagesPerPeriod(resourceUsage.getPublish().getMessagesPerPeriod());
+ p = recvdUsage.setReplicationDispatch();
+ p.setBytesPerPeriod(resourceUsage.getReplicationDispatch().getBytesPerPeriod());
+ p.setMessagesPerPeriod(resourceUsage.getReplicationDispatch().getMessagesPerPeriod());
+
recvdUsage.setStorage().setTotalBytes(resourceUsage.getStorage().getTotalBytes());
}
};
@@ -113,6 +117,8 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
assertNotNull(recvdUsage.getStorage());
assertEquals(recvdUsage.getPublish().getBytesPerPeriod(), 10001);
assertEquals(recvdUsage.getStorage().getTotalBytes(), 500003);
+ assertEquals(recvdUsage.getReplicationDispatch().getBytesPerPeriod(), 4000);
+ assertEquals(recvdUsage.getReplicationDispatch().getMessagesPerPeriod(), 2000);
}
private void prepareData() throws PulsarServerException, PulsarAdminException, PulsarClientException {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
index b97210b009a..4289b5ecca5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
@@ -18,21 +18,26 @@
*/
package org.apache.pulsar.broker.service;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.AssertJUnit.assertFalse;
import com.google.common.collect.Sets;
import java.lang.reflect.Method;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import lombok.Cleanup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.common.policies.data.DispatchRate;
+import org.apache.pulsar.common.policies.data.ResourceGroup;
import org.awaitility.Awaitility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,6 +103,7 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
// rate limiter disable by default
assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent());
+ assertFalse(topic.getReplicators().values().get(0).getResourceGroupDispatchRateLimiter().isPresent());
//set topic-level policy, which should take effect
DispatchRate topicRate = DispatchRate.builder()
@@ -119,6 +125,24 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1);
assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(),
-1L);
+
+ // ResourceGroupDispatchRateLimiter
+ String resourceGroupName = UUID.randomUUID().toString();
+ ResourceGroup resourceGroup = new ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInBytes(10L);
+ resourceGroup.setReplicationDispatchRateInMsgs(20L);
+ admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+ Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+ .getResourceGroup(resourceGroupName)));
+ admin1.topicPolicies().setResourceGroup(topicName, resourceGroupName);
+
+ Replicator replicator = topic.getReplicators().values().get(0);
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(replicator.getResourceGroupDispatchRateLimiter().isPresent());
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = replicator.getResourceGroupDispatchRateLimiter().get();
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+ });
}
@Test
@@ -142,6 +166,7 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
// rate limiter disable by default
assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent());
+ assertFalse(topic.getReplicators().values().get(0).getResourceGroupDispatchRateLimiter().isPresent());
//set namespace-level policy, which should take effect
DispatchRate topicRate = DispatchRate.builder()
@@ -163,6 +188,24 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1);
assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(),
-1L);
+
+ // ResourceGroupDispatchRateLimiter
+ String resourceGroupName = UUID.randomUUID().toString();
+ ResourceGroup resourceGroup = new ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInBytes(10L);
+ resourceGroup.setReplicationDispatchRateInMsgs(20L);
+ admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+ Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+ .getResourceGroup(resourceGroupName)));
+ admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName);
+
+ Replicator replicator = topic.getReplicators().values().get(0);
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(replicator.getResourceGroupDispatchRateLimiter().isPresent());
+ ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = replicator.getResourceGroupDispatchRateLimiter().get();
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+ assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+ });
}
@Test
@@ -545,5 +588,114 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
producer.close();
}
+ @Test
+ public void testResourceGroupReplicatorRateLimiter() throws Exception {
+ final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis();
+ final String topicName = "persistent://" + namespace + "/" + UUID.randomUUID();
+
+ admin1.namespaces().createNamespace(namespace);
+ // 0. set 2 clusters, there will be 1 replicator in each topic
+ admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
+
+ // ResourceGroupDispatchRateLimiter
+ int messageRate = 100;
+ String resourceGroupName = UUID.randomUUID().toString();
+ ResourceGroup resourceGroup = new ResourceGroup();
+ resourceGroup.setReplicationDispatchRateInMsgs((long) messageRate);
+ admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+ Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+ .getResourceGroup(resourceGroupName)));
+ admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName);
+
+ @Cleanup
+ PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+
+ @Cleanup
+ Producer<byte[]> producer = client1.newProducer().topic(topicName)
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
+
+ @Cleanup
+ PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
+ .build();
+ final AtomicInteger totalReceived = new AtomicInteger(0);
+
+ @Cleanup
+ Consumer<byte[]> consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
+
+ int numMessages = 500;
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(new byte[80]);
+ }
+
+ Assert.assertTrue(totalReceived.get() < messageRate * 2);
+ }
+
+ @Test
+ public void testReplicatorRateLimiterByBytes() throws Exception {
+ final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis();
+ final String topicName = "persistent://" + namespace + "/RateLimiterByBytes";
+
+ admin1.namespaces().createNamespace(namespace);
+ // 0. set 2 clusters, there will be 1 replicator in each topic
+ admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
+
+ final int byteRate = 400;
+ final int payloadSize = 100;
+ DispatchRate dispatchRate = DispatchRate.builder()
+ .dispatchThrottlingRateInMsg(-1)
+ .dispatchThrottlingRateInByte(byteRate)
+ .ratePeriodInSecond(360)
+ .build();
+ admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate);
+
+ @Cleanup
+ PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
+ @Cleanup
+ Producer<byte[]> producer = client1.newProducer().topic(topicName)
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
+
+ PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get();
+
+ Awaitility.await()
+ .untilAsserted(() -> assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()));
+ assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), byteRate);
+
+ @Cleanup
+ PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString())
+ .build();
+ final AtomicInteger totalReceived = new AtomicInteger(0);
+
+ @Cleanup
+ Consumer<byte[]> ignored = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2")
+ .messageListener((c1, msg) -> {
+ Assert.assertNotNull(msg, "Message cannot be null");
+ String receivedMessage = new String(msg.getData());
+ log.debug("Received message [{}] in the listener", receivedMessage);
+ totalReceived.incrementAndGet();
+ }).subscribe();
+
+ // The total bytes is 5 times the rate limit value.
+ int numMessages = byteRate / payloadSize * 5;
+ for (int i = 0; i < numMessages * payloadSize; i++) {
+ producer.send(new byte[payloadSize]);
+ }
+
+ Awaitility.await().pollDelay(5, TimeUnit.SECONDS)
+ .untilAsserted(() -> {
+ // The rate limit occurs in the next reading cycle, so a value fault tolerance needs to be added.
+ assertThat(totalReceived.get()).isLessThan((byteRate / payloadSize) + 2);
+ });
+ }
+
private static final Logger log = LoggerFactory.getLogger(ReplicatorRateLimiterTest.class);
}
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
index 843976eb13d..52eb9cfe317 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
@@ -75,6 +75,8 @@ public interface ResourceGroups {
* "PublishRateInBytes" : "value",
* "DispatchRateInMsgs" : "value",
* "DispatchRateInBytes" : "value"
+ * "ReplicationDispatchRateInMsgs" : "value"
+ * "ReplicationDispatchRateInBytes" : "value"
* </code>
* </pre>
*
@@ -101,6 +103,8 @@ public interface ResourceGroups {
* "PublishRateInBytes" : "value",
* "DispatchRateInMsgs" : "value",
* "DspatchRateInBytes" : "value"
+ * "ReplicationDispatchRateInMsgs" : "value"
+ * "ReplicationDispatchRateInBytes" : "value"
* </code>
* </pre>
*
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
index f6cd2a5a0ef..a771478f146 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
@@ -21,6 +21,8 @@ package org.apache.pulsar.client.admin;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride;
import org.apache.pulsar.common.policies.data.BacklogQuota;
@@ -1913,4 +1915,61 @@ public interface TopicPolicies {
* Topic name
*/
CompletableFuture<Void> removeAutoSubscriptionCreationAsync(String topic);
+
+ /**
+ * Get the ResourceGroup for a topic.
+ *
+ * @param topic Topic name
+ * @param applied True gets namespace level configuration if ResourceGroup does not exist on the topic.
+ * False gets topic level configuration.
+ * @return ResourceGroup
+ * @throws NotAuthorizedException Don't have admin permission
+ * @throws NotFoundException Topic does not exist
+ * @throws PulsarAdminException Unexpected error
+ */
+ String getResourceGroup(String topic, boolean applied) throws PulsarAdminException;
+
+ /**
+ * Get the ResourceGroup for a topic asynchronously.
+ *
+ * @param topic Topic name
+ */
+ CompletableFuture<String> getResourceGroupAsync(String topic, boolean applied);
+
+ /**
+ * Set the ResourceGroup for a topic.
+ *
+ * @param topic Topic name
+ * @param resourceGroupName ResourceGroup name
+ * @throws NotAuthorizedException Don't have admin permission
+ * @throws NotFoundException Topic does not exist
+ * @throws PulsarAdminException Unexpected error
+ */
+ void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException;
+
+ /**
+ * Set the ResourceGroup for a topic.
+ *
+ * @param topic Topic name
+ * @param resourceGroupName ResourceGroup name
+ */
+ CompletableFuture<Void> setResourceGroupAsync(String topic, String resourceGroupName);
+
+ /**
+ * Remove the ResourceGroup on a topic.
+ *
+ * @param topic Topic name
+ * @throws NotAuthorizedException Don't have admin permission
+ * @throws NotFoundException Topic does not exist
+ * @throws PulsarAdminException Unexpected error
+ */
+ void removeResourceGroup(String topic) throws PulsarAdminException;
+
+ /**
+ * Remove the ResourceGroup on a topic asynchronously.
+ *
+ * @param topic Topic name
+ */
+ CompletableFuture<Void> removeResourceGroupAsync(String topic);
}
+
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
index 24be2f9380b..870725021d4 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
@@ -35,6 +35,16 @@ public interface ReplicatorStats {
/** Total throughput delivered to the replication-subscriber (bytes/s). */
double getMsgThroughputOut();
+ /**
+ * Total bytes delivered to the replication-subscriber (bytes).
+ */
+ long getBytesOutCounter();
+
+ /**
+ * Total messages delivered to the replication-subscriber (msg).
+ */
+ long getMsgOutCounter();
+
/** Total rate of messages expired (msg/s). */
double getMsgRateExpired();
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
index bd47427e9aa..81e06107a8f 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
@@ -26,4 +26,7 @@ public class ResourceGroup {
private Long publishRateInBytes;
private Integer dispatchRateInMsgs;
private Long dispatchRateInBytes;
+
+ private Long replicationDispatchRateInMsgs;
+ private Long replicationDispatchRateInBytes;
}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
index 915b22a2589..1af249a969b 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.InvocationCallback;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
@@ -1260,6 +1261,54 @@ public class TopicPoliciesImpl extends BaseResource implements TopicPolicies {
return asyncDeleteRequest(path);
}
+ @Override
+ public String getResourceGroup(String topic, boolean applied) throws PulsarAdminException {
+ return sync(() -> getResourceGroupAsync(topic, applied));
+ }
+
+ @Override
+ public CompletableFuture<String> getResourceGroupAsync(String topic, boolean applied) {
+ TopicName tn = validateTopic(topic);
+ WebTarget path = topicPath(tn, "resourceGroup");
+ path = path.queryParam("applied", applied);
+ final CompletableFuture<String> future = new CompletableFuture<>();
+ asyncGetRequest(path,
+ new InvocationCallback<String>() {
+ @Override
+ public void completed(String rgName) {
+ future.complete(rgName);
+ }
+
+ @Override
+ public void failed(Throwable throwable) {
+ future.completeExceptionally(getApiException(throwable.getCause()));
+ }
+ });
+ return future;
+ }
+
+ @Override
+ public void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException {
+ sync(() -> setResourceGroupAsync(topic, resourceGroupName));
+ }
+
+ @Override
+ public CompletableFuture<Void> setResourceGroupAsync(String topic, String resourceGroupName) {
+ TopicName tn = validateTopic(topic);
+ WebTarget path = topicPath(tn, "resourceGroup");
+ return asyncPostRequest(path, Entity.entity(resourceGroupName, MediaType.APPLICATION_JSON_TYPE));
+ }
+
+ @Override
+ public void removeResourceGroup(String topic) throws PulsarAdminException {
+ sync(() -> removeResourceGroupAsync(topic));
+ }
+
+ @Override
+ public CompletableFuture<Void> removeResourceGroupAsync(String topic) {
+ return setResourceGroupAsync(topic, null);
+ }
+
/*
* returns topic name with encoded Local Name
*/
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
index 7ed853be44d..6e62f57218f 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
@@ -72,6 +72,14 @@ public class CmdResourceGroups extends CmdBase {
+ "(default -1 will be overwrite if not passed)", required = false)
private Long dispatchRateInBytes;
+ @Parameter(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate "
+ + "(default -1 will be overwrite if not passed)", required = false)
+ private Long replicationDispatchRateInMsgs;
+
+ @Parameter(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate "
+ + "(default -1 will be overwrite if not passed)", required = false)
+ private Long replicationDispatchRateInBytes;
+
@Override
void run() throws PulsarAdminException {
String name = getOneArgument(params);
@@ -81,6 +89,8 @@ public class CmdResourceGroups extends CmdBase {
resourcegroup.setDispatchRateInBytes(dispatchRateInBytes);
resourcegroup.setPublishRateInMsgs(publishRateInMsgs);
resourcegroup.setPublishRateInBytes(publishRateInBytes);
+ resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs);
+ resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes);
getAdmin().resourcegroups().createResourceGroup(name, resourcegroup);
}
}
@@ -107,6 +117,14 @@ public class CmdResourceGroups extends CmdBase {
"-bd" }, description = "byte-dispatch-rate ", required = false)
private Long dispatchRateInBytes;
+ @Parameter(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate "
+ + "(default -1 will be overwrite if not passed)", required = false)
+ private Long replicationDispatchRateInMsgs;
+
+ @Parameter(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate "
+ + "(default -1 will be overwrite if not passed)", required = false)
+ private Long replicationDispatchRateInBytes;
+
@Override
void run() throws PulsarAdminException {
String name = getOneArgument(params);
@@ -116,6 +134,8 @@ public class CmdResourceGroups extends CmdBase {
resourcegroup.setDispatchRateInBytes(dispatchRateInBytes);
resourcegroup.setPublishRateInMsgs(publishRateInMsgs);
resourcegroup.setPublishRateInBytes(publishRateInBytes);
+ resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs);
+ resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes);
getAdmin().resourcegroups().updateResourceGroup(name, resourcegroup);
}
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
index d567d0b3671..88c60963d3f 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
@@ -156,6 +156,10 @@ public class CmdTopicPolicies extends CmdBase {
jcommander.addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation());
jcommander.addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation());
jcommander.addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation());
+
+ jcommander.addCommand("set-resource-group", new SetResourceGroup());
+ jcommander.addCommand("get-resource-group", new GetResourceGroup());
+ jcommander.addCommand("remove-resource-group", new RemoveResourceGroup());
}
@Parameters(commandDescription = "Get entry filters for a topic")
@@ -1931,6 +1935,61 @@ public class CmdTopicPolicies extends CmdBase {
}
}
+ @Parameters(commandDescription = "Get ResourceGroup for a topic")
+ private class GetResourceGroup extends CliCommand {
+ @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+ private java.util.List<String> params;
+
+ @Parameter(names = {"--applied", "-a"}, description = "Get the applied policy of the topic")
+ private boolean applied = false;
+
+ @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+ + "If set to true, broker returned global topic policies")
+ private boolean isGlobal = false;
+
+ @Override
+ void run() throws PulsarAdminException {
+ String topic = validateTopicName(params);
+ print(getTopicPolicies(isGlobal).getResourceGroup(topic, applied));
+ }
+ }
+
+ @Parameters(commandDescription = "Set ResourceGroup for a topic")
+ private class SetResourceGroup extends CliCommand {
+ @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+ private java.util.List<String> params;
+
+ @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+ + "If set to true, broker returned global topic policies")
+ private boolean isGlobal = false;
+
+ @Parameter(names = {"--resource-group-name", "-rgn"}, description = "ResourceGroup name", required = true)
+ private String rgName;
+
+ @Override
+ void run() throws PulsarAdminException {
+ String topic = validateTopicName(params);
+ getTopicPolicies(isGlobal).setResourceGroup(topic, rgName);
+ }
+ }
+
+ @Parameters(commandDescription = "Remove ResourceGroup from a topic")
+ private class RemoveResourceGroup extends CliCommand {
+ @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+ private java.util.List<String> params;
+
+ @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+ + "If set to true, broker returned global topic policies")
+ private boolean isGlobal = false;
+
+
+ @Override
+ void run() throws PulsarAdminException {
+ String topic = validateTopicName(params);
+ getTopicPolicies(isGlobal).removeResourceGroup(topic);
+ }
+ }
+
private TopicPolicies getTopicPolicies(boolean isGlobal) {
return getAdmin().topicPolicies(isGlobal);
}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
index 0249272b72d..2607e14ffe7 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
@@ -60,6 +60,8 @@ public class HierarchyTopicPolicies {
final PolicyHierarchyValue<Boolean> schemaValidationEnforced;
final PolicyHierarchyValue<EntryFilters> entryFilters;
+ final PolicyHierarchyValue<String> resourceGroupName;
+
public HierarchyTopicPolicies() {
replicationClusters = new PolicyHierarchyValue<>();
retentionPolicies = new PolicyHierarchyValue<>();
@@ -90,5 +92,6 @@ public class HierarchyTopicPolicies {
dispatchRate = new PolicyHierarchyValue<>();
schemaValidationEnforced = new PolicyHierarchyValue<>();
entryFilters = new PolicyHierarchyValue<>();
+ resourceGroupName = new PolicyHierarchyValue<>();
}
}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
index 4a76170d116..2d62d09cfb1 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
@@ -86,6 +86,8 @@ public class TopicPolicies {
private Boolean schemaValidationEnforced;
+ private String resourceGroupName;
+
public boolean isGlobalPolicies() {
return isGlobal != null && isGlobal;
}
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
index 6933f5cc7ed..c744affbb7e 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
@@ -40,6 +40,16 @@ public class ReplicatorStatsImpl implements ReplicatorStats {
/** Total throughput delivered to the replication-subscriber (bytes/s). */
public double msgThroughputOut;
+ /**
+ * Total bytes delivered to the replication-subscriber (bytes).
+ */
+ public long bytesOutCounter;
+
+ /**
+ * Total messages delivered to the replication-subscriber (msg).
+ */
+ public long msgOutCounter;
+
/** Total rate of messages expired (msg/s). */
public double msgRateExpired;