You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by zi...@apache.org on 2024/10/29 03:55:49 UTC

(pulsar) 08/08: [feat][broker] Add ResourceGroup rate limit for topic dispatch and replication (#26)

This is an automated email from the ASF dual-hosted git repository.

zixuan pushed a commit to branch branch-as-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit a9672262041006b9674392b1c1784343a9c0ee7e
Author: Zixuan Liu <no...@gmail.com>
AuthorDate: Tue Oct 29 11:47:20 2024 +0800

    [feat][broker] Add ResourceGroup rate limit for topic dispatch and replication (#26)
    
    * [feat][broker] Add ResourceGroup rate limit for topic dispatch and replication
    
    Signed-off-by: Zixuan Liu <no...@gmail.com>
    
    * [fix][broker] Fix ownership loss
    
    Signed-off-by: Zixuan Liu <no...@gmail.com>
    
    * Fix test
    
    Signed-off-by: Zixuan Liu <no...@gmail.com>
    
    ---------
    
    Signed-off-by: Zixuan Liu <no...@gmail.com>
---
 .../apache/pulsar/broker/ServiceConfiguration.java |  27 ++
 .../broker/resources/ResourceGroupResources.java   |   4 +
 .../broker/admin/impl/PersistentTopicsBase.java    |  36 +++
 .../broker/admin/impl/ResourceGroupsBase.java      |  11 +-
 .../pulsar/broker/admin/v2/PersistentTopics.java   |  56 ++++
 .../pulsar/broker/namespace/OwnershipCache.java    |  12 +-
 .../pulsar/broker/resourcegroup/ResourceGroup.java | 175 ++++++++---
 .../ResourceGroupDispatchLimiter.java              | 172 +++++++++++
 .../ResourceGroupRateLimiterManager.java           |  67 +++++
 .../broker/resourcegroup/ResourceGroupService.java | 335 ++++++++++++++++-----
 .../resourcegroup/ResourceQuotaCalculatorImpl.java |  90 ++++--
 .../pulsar/broker/service/AbstractTopic.java       |  51 +++-
 .../apache/pulsar/broker/service/Replicator.java   |   5 +
 .../org/apache/pulsar/broker/service/Topic.java    |   5 +
 .../service/nonpersistent/NonPersistentTopic.java  |   6 +-
 .../persistent/GeoPersistentReplicator.java        |  28 +-
 .../PersistentDispatcherMultipleConsumers.java     |  22 ++
 .../PersistentDispatcherSingleActiveConsumer.java  |  25 ++
 .../service/persistent/PersistentReplicator.java   | 191 +++++++++---
 .../broker/service/persistent/PersistentTopic.java |  16 +-
 .../service/persistent/ShadowReplicator.java       |  28 +-
 pulsar-broker/src/main/proto/ResourceUsage.proto   |   1 +
 .../broker/admin/AdminResourceGroupTest.java       | 168 +++++++++++
 .../resourcegroup/ResourceGroupMetricTest.java     |  49 +++
 .../ResourceGroupRateLimiterManagerTest.java       | 109 +++++++
 .../resourcegroup/ResourceGroupServiceTest.java    | 213 ++++++++++++-
 ...ourceGroupUsageAggregationOnTopicLevelTest.java | 228 ++++++++++++++
 .../ResourceGroupUsageAggregationTest.java         |   2 -
 .../ResourceQuotaCalculatorImplTest.java           |  13 +-
 .../ResourceUsageTransportManagerTest.java         |   8 +-
 .../broker/service/ReplicatorRateLimiterTest.java  | 152 ++++++++++
 .../apache/pulsar/client/admin/ResourceGroups.java |   4 +
 .../apache/pulsar/client/admin/TopicPolicies.java  |  59 ++++
 .../common/policies/data/ReplicatorStats.java      |  10 +
 .../pulsar/common/policies/data/ResourceGroup.java |   3 +
 .../client/admin/internal/TopicPoliciesImpl.java   |  49 +++
 .../apache/pulsar/admin/cli/CmdResourceGroups.java |  20 ++
 .../apache/pulsar/admin/cli/CmdTopicPolicies.java  |  59 ++++
 .../policies/data/HierarchyTopicPolicies.java      |   3 +
 .../pulsar/common/policies/data/TopicPolicies.java |   2 +
 .../policies/data/stats/ReplicatorStatsImpl.java   |  10 +
 41 files changed, 2307 insertions(+), 217 deletions(-)

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ff401fdb2b7..4cbc3bfc409 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1109,6 +1109,33 @@ public class ServiceConfiguration implements PulsarConfiguration {
     )
     private int subscriptionPatternMaxLength = 50;
 
+    @FieldContext(
+            dynamic = true,
+            category = CATEGORY_POLICIES,
+            doc = "The percentage difference that is considered \"within limits\" to suppress usage reporting"
+                    + "Setting this to 0 will also make us report in every round."
+    )
+    private int resourceUsageReportSuppressionTolerancePercentage = 5;
+
+    @FieldContext(
+            category = CATEGORY_POLICIES,
+            doc = "The maximum number of successive rounds that we can suppress reporting local usage, because there "
+                    + "was no substantial change from the prior round. This is to ensure the reporting does not "
+                    + "become too chatty. Set this value to one more than the cadence of sending reports; e.g., if "
+                    + "you want to send every 3rd report, set the value to 4."
+                    + "Setting this to 0 will make us report in every round."
+                    + "Don't set to negative values; behavior will be disabled"
+    )
+    private int resourceUsageMaxUsageReportSuppressRounds = 5;
+
+    @FieldContext(
+            dynamic = true,
+            category = CATEGORY_POLICIES,
+            doc = "ResourceGroup rate limit will be triggered when the total traffic exceeds the product of the "
+                    + "rate-limit value and the threshold. Value range: [0,1]."
+    )
+    private double resourceGroupLocalQuotaThreshold = 0;
+
     // <-- dispatcher read settings -->
     @FieldContext(
         dynamic = true,
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
index 578609b2195..562261ff7ac 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/ResourceGroupResources.java
@@ -46,6 +46,10 @@ public class ResourceGroupResources extends BaseResources<ResourceGroup> {
         return exists(joinPath(BASE_PATH, resourceGroupName));
     }
 
+    public CompletableFuture<Boolean> resourceGroupExistsAsync(String resourceGroupName) {
+        return existsAsync(joinPath(BASE_PATH, resourceGroupName));
+    }
+
     public void createResourceGroup(String resourceGroupName, ResourceGroup rg) throws MetadataStoreException {
         create(joinPath(BASE_PATH, resourceGroupName), rg);
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index abd9a37aaf7..65ef2a10a0e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -5682,4 +5682,40 @@ public class PersistentTopicsBase extends AdminResource {
                             return null;
                         }));
     }
+
+    protected CompletableFuture<Void> internalSetResourceGroup(String resourceGroupName, boolean isGlobal) {
+        boolean isDelete = StringUtils.isEmpty(resourceGroupName);
+        return validateTopicPolicyOperationAsync(topicName, PolicyName.RESOURCEGROUP, PolicyOperation.WRITE)
+                .thenCompose(__ -> {
+                    if (isDelete) {
+                        return CompletableFuture.completedFuture(true);
+                    }
+                    return resourceGroupResources().resourceGroupExistsAsync(resourceGroupName);
+                })
+                .thenCompose(exists -> {
+                    if (!exists) {
+                        return FutureUtil.failedFuture(new RestException(Status.NOT_FOUND,
+                                "ResourceGroup does not exist"));
+                    }
+                    return getTopicPoliciesAsyncWithRetry(topicName, isGlobal).thenCompose(op -> {
+                        TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
+                        topicPolicies.setResourceGroupName(isDelete ? null : resourceGroupName);
+                        topicPolicies.setIsGlobal(isGlobal);
+                        return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
+                    });
+                });
+    }
+
+    protected CompletableFuture<String> internalGetResourceGroup(boolean applied, boolean isGlobal) {
+        return validateTopicPolicyOperationAsync(topicName, PolicyName.RESOURCEGROUP, PolicyOperation.READ)
+                .thenCompose(__ -> getTopicPoliciesAsyncWithRetry(topicName, isGlobal)
+                        .thenApply(op -> op.map(TopicPolicies::getResourceGroupName)
+                                .orElseGet(() -> {
+                                    if (applied) {
+                                        return getNamespacePolicies(namespaceName).resource_group_name;
+                                    }
+                                    return null;
+                                })
+                        ));
+    }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
index 826b9c322e3..9f984a35e11 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ResourceGroupsBase.java
@@ -75,7 +75,12 @@ public abstract class ResourceGroupsBase extends AdminResource {
             if (rgConfig.getDispatchRateInBytes() != null) {
                 resourceGroup.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes());
             }
-
+            if (rgConfig.getReplicationDispatchRateInBytes() != null) {
+                resourceGroup.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes());
+            }
+            if (rgConfig.getReplicationDispatchRateInMsgs() != null) {
+                resourceGroup.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs());
+            }
             // write back the new ResourceGroup config.
             resourceGroupResources().updateResourceGroup(rgName, r -> resourceGroup);
             log.info("[{}] Successfully updated the ResourceGroup {}", clientAppId(), rgName);
@@ -96,6 +101,10 @@ public abstract class ResourceGroupsBase extends AdminResource {
                 ? -1 : rgConfig.getDispatchRateInMsgs());
         rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes() == null
                 ? -1 : rgConfig.getDispatchRateInBytes());
+        rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes() == null
+                ? -1 : rgConfig.getReplicationDispatchRateInBytes());
+        rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs() == null
+                ? -1 : rgConfig.getReplicationDispatchRateInMsgs());
         try {
             resourceGroupResources().createResourceGroup(rgName, rgConfig);
             log.info("[{}] Created ResourceGroup {}", clientAppId(), rgName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 2e1b943a2cc..bee27423228 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -4888,5 +4888,61 @@ public class PersistentTopics extends PersistentTopicsBase {
                 });
     }
 
+    @POST
+    @Path("/{tenant}/{namespace}/{topic}/resourceGroup")
+    @ApiOperation(value = "Set ResourceGroup for a topic")
+    @ApiResponses(value = {
+            @ApiResponse(code = 403, message = "Don't have admin permission"),
+            @ApiResponse(code = 404, message = "Topic doesn't exist"),
+            @ApiResponse(code = 405, message =
+                    "Topic level policy is disabled, enable the topic level policy and retry"),
+            @ApiResponse(code = 409, message = "Concurrent modification")
+    })
+    public void setResourceGroup(
+            @Suspended final AsyncResponse asyncResponse,
+            @PathParam("tenant") String tenant,
+            @PathParam("namespace") String namespace,
+            @PathParam("topic") String encodedTopic,
+            @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal,
+            @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
+            @ApiParam(value = "ResourceGroup name", required = true) String resourceGroupName) {
+        validateTopicName(tenant, namespace, encodedTopic);
+        preValidation(authoritative)
+                .thenCompose(__ -> internalSetResourceGroup(resourceGroupName, isGlobal))
+                .thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
+                .exceptionally(ex -> {
+                    handleTopicPolicyException("setResourceGroup", ex, asyncResponse);
+                    return null;
+                });
+    }
+
+    @GET
+    @Path("/{tenant}/{namespace}/{topic}/resourceGroup")
+    @ApiOperation(value = "Get ResourceGroup for a topic")
+    @ApiResponses(value = {
+            @ApiResponse(code = 403, message = "Don't have admin permission"),
+            @ApiResponse(code = 404, message = "Topic doesn't exist"),
+            @ApiResponse(code = 405, message =
+                    "Topic level policy is disabled, enable the topic level policy and retry"),
+            @ApiResponse(code = 409, message = "Concurrent modification")
+    })
+    public void getResourceGroup(
+            @Suspended final AsyncResponse asyncResponse,
+            @PathParam("tenant") String tenant,
+            @PathParam("namespace") String namespace,
+            @PathParam("topic") String encodedTopic,
+            @QueryParam("applied") @DefaultValue("false") boolean applied,
+            @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal,
+            @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
+        validateTopicName(tenant, namespace, encodedTopic);
+        preValidation(authoritative)
+                .thenCompose(__ -> internalGetResourceGroup(applied, isGlobal))
+                .thenApply(asyncResponse::resume)
+                .exceptionally(ex -> {
+                    handleTopicPolicyException("getResourceGroup", ex, asyncResponse);
+                    return null;
+                });
+    }
+
     private static final Logger log = LoggerFactory.getLogger(PersistentTopics.class);
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
index 0033abf36c7..553fc95e9dc 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
@@ -172,7 +172,17 @@ public class OwnershipCache {
 
         // If we're not the owner, we need to check if anybody else is
         String path = ServiceUnitUtils.path(suName);
-        return lockManager.readLock(path);
+        return lockManager.readLock(path).thenCompose(owner -> {
+            //If the current broker is the owner, we must try to acquire ownership to avoid cache loss.
+            if (owner.isPresent() && owner.get().equals(selfOwnerInfo)) {
+                try {
+                    return tryAcquiringOwnership(suName).thenApply(Optional::ofNullable);
+                } catch (Exception e) {
+                    return CompletableFuture.failedFuture(e);
+                }
+            }
+            return CompletableFuture.completedFuture(owner);
+        });
     }
 
     /**
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
index 96a3a9d44b9..9a7e20a92e4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroup.java
@@ -18,15 +18,20 @@
  */
 package org.apache.pulsar.broker.resourcegroup;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.annotations.VisibleForTesting;
 import io.prometheus.client.Counter;
-import java.util.HashMap;
+import io.prometheus.client.Gauge;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import lombok.Getter;
+import lombok.ToString;
 import lombok.val;
+import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupOpStatus;
 import org.apache.pulsar.broker.service.resource.usage.NetworkUsage;
 import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
@@ -49,6 +54,7 @@ public class ResourceGroup {
     /**
      * Convenience class for bytes and messages counts, which are used together in a lot of the following code.
      */
+    @ToString
     public static class BytesAndMessagesCount {
         public long bytes;
         public long messages;
@@ -61,6 +67,7 @@ public class ResourceGroup {
     public enum ResourceGroupMonitoringClass {
         Publish,
         Dispatch,
+        ReplicationDispatch,
         // Storage;  // Punt this for now, until we have a clearer idea of the usage, statistics, etc.
     }
 
@@ -70,7 +77,7 @@ public class ResourceGroup {
     public enum ResourceGroupRefTypes {
         Tenants,
         Namespaces,
-        // Topics;  // Punt this for when we support direct ref/under from topics.
+        Topics
     }
 
     // Default ctor: it is not expected that anything outside of this package will need to directly
@@ -85,6 +92,13 @@ public class ResourceGroup {
         this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor());
         log.info("attaching publish rate limiter {} to {} get {}", this.resourceGroupPublishLimiter, name,
           this.getResourceGroupPublishLimiter());
+        this.resourceGroupReplicationDispatchLimiter = ResourceGroupRateLimiterManager
+                .newReplicationDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+        log.info("attaching replication dispatch rate limiter {} to {}", this.resourceGroupReplicationDispatchLimiter,
+                name);
+        this.resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager
+                .newDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+        log.info("attaching topic dispatch rate limiter {} to {}", this.resourceGroupDispatchLimiter, name);
     }
 
     // ctor for overriding the transport-manager fill/set buffer.
@@ -98,6 +112,10 @@ public class ResourceGroup {
         this.setResourceGroupMonitoringClassFields();
         this.setResourceGroupConfigParameters(rgConfig);
         this.resourceGroupPublishLimiter = new ResourceGroupPublishLimiter(rgConfig, rgs.getPulsar().getExecutor());
+        this.resourceGroupReplicationDispatchLimiter = ResourceGroupRateLimiterManager
+                .newReplicationDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
+        this.resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager
+                .newDispatchRateLimiter(rgConfig, rgs.getPulsar().getExecutor());
         this.ruPublisher = rgPublisher;
         this.ruConsumer = rgConsumer;
     }
@@ -108,12 +126,15 @@ public class ResourceGroup {
         this.resourceGroupName = other.resourceGroupName;
         this.rgs = other.rgs;
         this.resourceGroupPublishLimiter = other.resourceGroupPublishLimiter;
+        this.resourceGroupReplicationDispatchLimiter = other.resourceGroupReplicationDispatchLimiter;
+        this.resourceGroupDispatchLimiter = other.resourceGroupDispatchLimiter;
         this.setResourceGroupMonitoringClassFields();
 
         // ToDo: copy the monitoring class fields, and ruPublisher/ruConsumer from other, if required.
 
         this.resourceGroupNamespaceRefs = other.resourceGroupNamespaceRefs;
         this.resourceGroupTenantRefs = other.resourceGroupTenantRefs;
+        this.resourceGroupTopicRefs = other.resourceGroupTopicRefs;
 
         for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) {
             PerMonitoringClassFields thisFields = this.monitoringClassFields[idx];
@@ -146,12 +167,19 @@ public class ResourceGroup {
         pubBmc.messages = rgConfig.getPublishRateInMsgs();
         pubBmc.bytes = rgConfig.getPublishRateInBytes();
         this.resourceGroupPublishLimiter.update(pubBmc);
+        ResourceGroupRateLimiterManager
+                .updateReplicationDispatchRateLimiter(resourceGroupReplicationDispatchLimiter, rgConfig);
+        ResourceGroupRateLimiterManager.updateDispatchRateLimiter(resourceGroupDispatchLimiter, rgConfig);
     }
 
     protected long getResourceGroupNumOfNSRefs() {
         return this.resourceGroupNamespaceRefs.size();
     }
 
+    protected long getResourceGroupNumOfTopicRefs() {
+        return this.resourceGroupTopicRefs.size();
+    }
+
     protected long getResourceGroupNumOfTenantRefs() {
         return this.resourceGroupTenantRefs.size();
     }
@@ -169,6 +197,9 @@ public class ResourceGroup {
             case Namespaces:
                 set = this.resourceGroupNamespaceRefs;
                 break;
+            case Topics:
+                set = this.resourceGroupTopicRefs;
+                break;
         }
 
         if (ref) {
@@ -179,7 +210,8 @@ public class ResourceGroup {
             set.add(name);
 
             // If this is the first ref, register with the transport manager.
-            if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 1) {
+            if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size()
+                    + this.resourceGroupTopicRefs.size() == 1) {
                 if (log.isDebugEnabled()) {
                     log.debug("registerUsage for RG={}: registering with transport-mgr", this.resourceGroupName);
                 }
@@ -194,7 +226,8 @@ public class ResourceGroup {
             set.remove(name);
 
             // If this was the last ref, unregister from the transport manager.
-            if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size() == 0) {
+            if (this.resourceGroupTenantRefs.size() + this.resourceGroupNamespaceRefs.size()
+                    + this.resourceGroupTopicRefs.size() == 0) {
                 if (log.isDebugEnabled()) {
                     log.debug("unRegisterUsage for RG={}: un-registering from transport-mgr", this.resourceGroupName);
                 }
@@ -226,6 +259,11 @@ public class ResourceGroup {
             resourceUsage.clearDispatch();
         }
 
+        p = resourceUsage.setReplicationDispatch();
+        if (!this.setUsageInMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch, p)) {
+            resourceUsage.clearReplicationDispatch();
+        }
+
         // Punt storage for now.
     }
 
@@ -239,6 +277,11 @@ public class ResourceGroup {
             this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.Dispatch, resourceUsage.getDispatch(),
                     broker);
         }
+
+        if (resourceUsage.hasReplicationDispatch()) {
+            this.getUsageFromMonitoredEntity(ResourceGroupMonitoringClass.ReplicationDispatch,
+                    resourceUsage.getReplicationDispatch(), broker);
+        }
         // Punt storage for now.
     }
 
@@ -319,7 +362,7 @@ public class ResourceGroup {
 
         monEntity.usageFromOtherBrokersLock.lock();
         try {
-            pbus = monEntity.usageFromOtherBrokers.get(myBrokerId);
+            pbus = monEntity.usageFromOtherBrokers.getIfPresent(myBrokerId);
         } finally {
             monEntity.usageFromOtherBrokersLock.unlock();
         }
@@ -343,7 +386,7 @@ public class ResourceGroup {
         monEntity.usageFromOtherBrokersLock.lock();
         BytesAndMessagesCount retStats = new BytesAndMessagesCount();
         try {
-            monEntity.usageFromOtherBrokers.forEach((broker, brokerUsage) -> {
+            monEntity.usageFromOtherBrokers.asMap().forEach((broker, brokerUsage) -> {
                 retStats.bytes += brokerUsage.usedValues.bytes;
                 retStats.messages += brokerUsage.usedValues.messages;
             });
@@ -356,14 +399,6 @@ public class ResourceGroup {
 
     protected BytesAndMessagesCount updateLocalQuota(ResourceGroupMonitoringClass monClass,
                                                      BytesAndMessagesCount newQuota) throws PulsarAdminException {
-        // Only the Publish side is functional now; add the Dispatch side code when the consume side is ready.
-        if (!ResourceGroupMonitoringClass.Publish.equals(monClass)) {
-            if (log.isDebugEnabled()) {
-                log.debug("Doing nothing for monClass={}; only Publish is functional", monClass);
-            }
-            return null;
-        }
-
         this.checkMonitoringClass(monClass);
         BytesAndMessagesCount oldBMCount;
 
@@ -372,7 +407,22 @@ public class ResourceGroup {
         oldBMCount = monEntity.quotaForNextPeriod;
         try {
             monEntity.quotaForNextPeriod = newQuota;
-            this.resourceGroupPublishLimiter.update(newQuota);
+            switch (monClass) {
+                case ReplicationDispatch:
+                    ResourceGroupRateLimiterManager
+                            .updateReplicationDispatchRateLimiter(resourceGroupReplicationDispatchLimiter, newQuota);
+                    break;
+                case Publish:
+                    this.resourceGroupPublishLimiter.update(newQuota);
+                    break;
+                case Dispatch:
+                    ResourceGroupRateLimiterManager.updateDispatchRateLimiter(resourceGroupDispatchLimiter, newQuota);
+                    break;
+                default:
+                    if (log.isDebugEnabled()) {
+                        log.debug("Doing nothing for monClass={};", monClass);
+                    }
+            }
         } finally {
             monEntity.localUsageStatsLock.unlock();
         }
@@ -424,9 +474,16 @@ public class ResourceGroup {
     }
 
     private void checkMonitoringClass(ResourceGroupMonitoringClass monClass) throws PulsarAdminException {
-        if (monClass != ResourceGroupMonitoringClass.Publish && monClass != ResourceGroupMonitoringClass.Dispatch) {
-            String errMesg = "Unexpected monitoring class: " + monClass;
-            throw new PulsarAdminException(errMesg);
+        switch (monClass) {
+            case Publish:
+                break;
+            case Dispatch:
+                break;
+            case ReplicationDispatch:
+                break;
+            default:
+                String errMesg = "Unexpected monitoring class: " + monClass;
+                throw new PulsarAdminException(errMesg);
         }
     }
 
@@ -501,7 +558,7 @@ public class ResourceGroup {
         long newByteCount, newMessageCount;
 
         monEntity = this.monitoringClassFields[idx];
-        usageStats = monEntity.usageFromOtherBrokers.get(broker);
+        usageStats = monEntity.usageFromOtherBrokers.getIfPresent(broker);
         if (usageStats == null) {
             usageStats = new PerBrokerUsageStats();
             usageStats.usedValues = new BytesAndMessagesCount();
@@ -513,13 +570,15 @@ public class ResourceGroup {
             newMessageCount = p.getMessagesPerPeriod();
             usageStats.usedValues.messages = newMessageCount;
             usageStats.lastResourceUsageReadTimeMSecsSinceEpoch = System.currentTimeMillis();
-            oldUsageStats = monEntity.usageFromOtherBrokers.put(broker, usageStats);
+            oldUsageStats = monEntity.usageFromOtherBrokers.getIfPresent(broker);
+            monEntity.usageFromOtherBrokers.put(broker, usageStats);
         } finally {
             monEntity.usageFromOtherBrokersLock.unlock();
         }
         rgRemoteUsageReportsBytes.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newByteCount);
         rgRemoteUsageReportsMessages.labels(this.ruConsumer.getID(), monClass.name(), broker).inc(newMessageCount);
-
+        rgRemoteUsageReportsBytesGauge.labels(this.ruConsumer.getID(), monClass.name(), broker).set(newByteCount);
+        rgRemoteUsageReportsMessagesGauge.labels(this.ruConsumer.getID(), monClass.name(), broker).set(newMessageCount);
         oldByteCount = oldMessageCount = -1;
         if (oldUsageStats != null) {
             oldByteCount = oldUsageStats.usedValues.bytes;
@@ -536,21 +595,18 @@ public class ResourceGroup {
     }
 
     private void setResourceGroupMonitoringClassFields() {
-        PerMonitoringClassFields monClassFields;
+        ServiceConfiguration conf = rgs.getPulsar().getConfiguration();
+        long resourceUsageTransportPublishIntervalInSecs = conf.getResourceUsageTransportPublishIntervalInSecs();
+        int maxUsageReportSuppressRounds = Math.max(conf.getResourceUsageMaxUsageReportSuppressRounds(), 1);
+        long cacheInMS = TimeUnit.SECONDS.toMillis(
+                resourceUsageTransportPublishIntervalInSecs * maxUsageReportSuppressRounds * 2);
+        // Usage report data is cached to the memory, when the broker is restart or offline, we need an elimination
+        // strategy to release the quota occupied by other broker.
+        //
+        // Considering that each broker starts at a different time, the cache time should be equal to the mandatory
+        // reporting period * 2.
         for (int idx = 0; idx < ResourceGroupMonitoringClass.values().length; idx++) {
-            this.monitoringClassFields[idx] = new PerMonitoringClassFields();
-
-            monClassFields = this.monitoringClassFields[idx];
-            monClassFields.configValuesPerPeriod = new BytesAndMessagesCount();
-            monClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount();
-            monClassFields.lastReportedValues = new BytesAndMessagesCount();
-            monClassFields.quotaForNextPeriod = new BytesAndMessagesCount();
-            monClassFields.totalUsedLocally = new BytesAndMessagesCount();
-            monClassFields.usageFromOtherBrokers = new HashMap<>();
-
-            monClassFields.usageFromOtherBrokersLock = new ReentrantLock();
-            // ToDo: Change the following to a ReadWrite lock if needed.
-            monClassFields.localUsageStatsLock = new ReentrantLock();
+            this.monitoringClassFields[idx] = PerMonitoringClassFields.create(cacheInMS);
         }
     }
 
@@ -568,6 +624,14 @@ public class ResourceGroup {
                 ? -1 : rgConfig.getDispatchRateInBytes();
         this.monitoringClassFields[idx].configValuesPerPeriod.messages = rgConfig.getDispatchRateInMsgs() == null
                 ? -1 : rgConfig.getDispatchRateInMsgs();
+
+        idx = ResourceGroupMonitoringClass.ReplicationDispatch.ordinal();
+        this.monitoringClassFields[idx]
+                .configValuesPerPeriod.bytes = rgConfig.getReplicationDispatchRateInBytes() == null
+                ? -1 : rgConfig.getReplicationDispatchRateInBytes();
+        this.monitoringClassFields[idx]
+                .configValuesPerPeriod.messages = rgConfig.getReplicationDispatchRateInMsgs() == null
+                ? -1 : rgConfig.getReplicationDispatchRateInMsgs();
     }
 
     private void setDefaultResourceUsageTransportHandlers() {
@@ -613,6 +677,7 @@ public class ResourceGroup {
     // across all of its usage classes (publish/dispatch/...).
     private Set<String> resourceGroupTenantRefs = ConcurrentHashMap.newKeySet();
     private Set<String> resourceGroupNamespaceRefs = ConcurrentHashMap.newKeySet();
+    private Set<String> resourceGroupTopicRefs = ConcurrentHashMap.newKeySet();
 
     // Blobs required for transport manager's resource-usage register/unregister ops.
     ResourceUsageConsumer ruConsumer;
@@ -631,11 +696,21 @@ public class ResourceGroup {
             .help("Bytes used reported about this <RG, monitoring class> from a remote broker")
             .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
             .register();
+    private static final Gauge rgRemoteUsageReportsBytesGauge = Gauge.build()
+            .name("pulsar_resource_group_remote_usage_bytes_used_gauge")
+            .help("Bytes used reported about this <RG, monitoring class> from a remote broker")
+            .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
+            .register();
     private static final Counter rgRemoteUsageReportsMessages = Counter.build()
             .name("pulsar_resource_group_remote_usage_messages_used")
             .help("Messages used reported about this <RG, monitoring class> from a remote broker")
             .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
             .register();
+    private static final Gauge rgRemoteUsageReportsMessagesGauge = Gauge.build()
+            .name("pulsar_resource_group_remote_usage_messages_used_gauge")
+            .help("Messages used reported about this <RG, monitoring class> from a remote broker")
+            .labelNames(resourceGroupMontoringclassRemotebrokerLabels)
+            .register();
 
     private static final Counter rgLocalUsageReportCount = Counter.build()
             .name("pulsar_resource_group_local_usage_reported")
@@ -647,6 +722,12 @@ public class ResourceGroup {
     @Getter
     protected ResourceGroupPublishLimiter resourceGroupPublishLimiter;
 
+    @Getter
+    protected ResourceGroupDispatchLimiter resourceGroupReplicationDispatchLimiter;
+
+    @Getter
+    protected ResourceGroupDispatchLimiter resourceGroupDispatchLimiter;
+
     protected static class PerMonitoringClassFields {
         // This lock covers all the "local" counts (i.e., except for the per-broker usage stats).
         Lock localUsageStatsLock;
@@ -671,11 +752,33 @@ public class ResourceGroup {
         int numSuppressedUsageReports;
 
         // Accumulated stats of local usage.
+        @VisibleForTesting
         BytesAndMessagesCount totalUsedLocally;
 
         // This lock covers all the non-local usage counts, received from other brokers.
         Lock usageFromOtherBrokersLock;
-        public HashMap<String, PerBrokerUsageStats> usageFromOtherBrokers;
+        public Cache<String, PerBrokerUsageStats> usageFromOtherBrokers;
+
+        private PerMonitoringClassFields(){
+
+        }
+
+        static PerMonitoringClassFields create(long durationMs) {
+            PerMonitoringClassFields perMonitoringClassFields = new PerMonitoringClassFields();
+            perMonitoringClassFields.configValuesPerPeriod = new BytesAndMessagesCount();
+            perMonitoringClassFields.usedLocallySinceLastReport = new BytesAndMessagesCount();
+            perMonitoringClassFields.lastReportedValues = new BytesAndMessagesCount();
+            perMonitoringClassFields.quotaForNextPeriod = new BytesAndMessagesCount();
+            perMonitoringClassFields.totalUsedLocally = new BytesAndMessagesCount();
+            perMonitoringClassFields.usageFromOtherBrokersLock = new ReentrantLock();
+            // ToDo: Change the following to a ReadWrite lock if needed.
+            perMonitoringClassFields.localUsageStatsLock = new ReentrantLock();
+
+            perMonitoringClassFields.usageFromOtherBrokers = Caffeine.newBuilder()
+                    .expireAfterWrite(durationMs, TimeUnit.MILLISECONDS)
+                    .build();
+            return perMonitoringClassFields;
+        }
     }
 
     // Usage stats for this RG obtained from other brokers.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java
new file mode 100644
index 00000000000..db2a9045511
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupDispatchLimiter.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.pulsar.common.util.RateLimiter;
+
+public class ResourceGroupDispatchLimiter implements AutoCloseable {
+
+    private final ScheduledExecutorService executorService;
+    private volatile RateLimiter dispatchRateLimiterOnMessage;
+    private volatile RateLimiter dispatchRateLimiterOnByte;
+
+    public ResourceGroupDispatchLimiter(ScheduledExecutorService executorService,
+                                        long dispatchRateInMsgs, long dispatchRateInBytes) {
+        this.executorService = executorService;
+        update(dispatchRateInMsgs, dispatchRateInBytes);
+    }
+
+    public void update(long dispatchRateInMsgs, long dispatchRateInBytes) {
+        if (dispatchRateInMsgs > 0) {
+            if (dispatchRateLimiterOnMessage != null) {
+                this.dispatchRateLimiterOnMessage.setRate(dispatchRateInMsgs);
+            } else {
+                this.dispatchRateLimiterOnMessage =
+                        RateLimiter.builder()
+                                .scheduledExecutorService(executorService)
+                                .permits(dispatchRateInMsgs)
+                                .rateTime(1)
+                                .timeUnit(TimeUnit.SECONDS)
+                                .permitUpdater(null)
+                                .isDispatchOrPrecisePublishRateLimiter(true)
+                                .build();
+            }
+        } else {
+            if (this.dispatchRateLimiterOnMessage != null) {
+                this.dispatchRateLimiterOnMessage.close();
+                this.dispatchRateLimiterOnMessage = null;
+            }
+        }
+
+        if (dispatchRateInBytes > 0) {
+            if (dispatchRateLimiterOnByte != null) {
+                this.dispatchRateLimiterOnByte.setRate(dispatchRateInBytes);
+            } else {
+                this.dispatchRateLimiterOnByte =
+                        RateLimiter.builder()
+                                .scheduledExecutorService(executorService)
+                                .permits(dispatchRateInBytes)
+                                .rateTime(1)
+                                .timeUnit(TimeUnit.SECONDS)
+                                .permitUpdater(null)
+                                .isDispatchOrPrecisePublishRateLimiter(true)
+                                .build();
+            }
+        } else {
+            if (this.dispatchRateLimiterOnByte != null) {
+                this.dispatchRateLimiterOnByte.close();
+                this.dispatchRateLimiterOnByte = null;
+            }
+        }
+    }
+
+    /**
+     * returns available msg-permit if msg-dispatch-throttling is enabled else it returns -1.
+     *
+     * @return
+     */
+    public long getAvailableDispatchRateLimitOnMsg() {
+        return dispatchRateLimiterOnMessage == null ? -1 : dispatchRateLimiterOnMessage.getAvailablePermits();
+    }
+
+    /**
+     * returns available byte-permit if msg-dispatch-throttling is enabled else it returns -1.
+     *
+     * @return
+     */
+    public long getAvailableDispatchRateLimitOnByte() {
+        return dispatchRateLimiterOnByte == null ? -1 : dispatchRateLimiterOnByte.getAvailablePermits();
+    }
+
+    /**
+     * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed.
+     *
+     * @param numberOfMessages
+     * @param byteSize
+     */
+    public void consumeDispatchQuota(long numberOfMessages, long byteSize) {
+        if (numberOfMessages > 0 && dispatchRateLimiterOnMessage != null) {
+            dispatchRateLimiterOnMessage.tryAcquire(numberOfMessages);
+        }
+        if (byteSize > 0 && dispatchRateLimiterOnByte != null) {
+            dispatchRateLimiterOnByte.tryAcquire(byteSize);
+        }
+    }
+
+    /**
+     * It acquires msg and bytes permits from rate-limiter and returns if acquired permits succeed.
+     *
+     * @param numberOfMessages
+     * @param byteSize
+     */
+    public boolean tryAcquire(long numberOfMessages, long byteSize) {
+        if (numberOfMessages > 0 && dispatchRateLimiterOnMessage != null) {
+            if (!dispatchRateLimiterOnMessage.tryAcquire(numberOfMessages)) {
+                return false;
+            }
+        }
+        if (byteSize > 0 && dispatchRateLimiterOnByte != null) {
+            if (!dispatchRateLimiterOnByte.tryAcquire(byteSize)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Checks if dispatch-rate limiting is enabled.
+     *
+     * @return
+     */
+    public boolean isDispatchRateLimitingEnabled() {
+        return dispatchRateLimiterOnMessage != null || dispatchRateLimiterOnByte != null;
+    }
+
+    public void close() {
+        if (dispatchRateLimiterOnMessage != null) {
+            dispatchRateLimiterOnMessage = null;
+        }
+        if (dispatchRateLimiterOnByte != null) {
+            dispatchRateLimiterOnByte = null;
+        }
+    }
+
+    /**
+     * Get configured msg dispatch-throttling rate. Returns -1 if not configured
+     *
+     * @return
+     */
+    public long getDispatchRateOnMsg() {
+        return dispatchRateLimiterOnMessage != null ? dispatchRateLimiterOnMessage.getRate() : -1;
+    }
+
+    /**
+     * Get configured byte dispatch-throttling rate. Returns -1 if not configured
+     *
+     * @return
+     */
+    public long getDispatchRateOnByte() {
+        return dispatchRateLimiterOnByte != null ? dispatchRateLimiterOnByte.getRate() : -1;
+    }
+
+
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java
new file mode 100644
index 00000000000..2011ad12974
--- /dev/null
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManager.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import java.util.Optional;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+
+public class ResourceGroupRateLimiterManager {
+
+    static ResourceGroupDispatchLimiter newReplicationDispatchRateLimiter(
+            org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup,
+            ScheduledExecutorService executorService) {
+        long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L);
+        long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L);
+        return new ResourceGroupDispatchLimiter(executorService, msgs, bytes);
+    }
+
+    static void updateReplicationDispatchRateLimiter(
+            ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+            org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) {
+        long msgs = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInMsgs()).orElse(-1L);
+        long bytes = Optional.ofNullable(resourceGroup.getReplicationDispatchRateInBytes()).orElse(-1L);
+        resourceGroupDispatchLimiter.update(msgs, bytes);
+    }
+
+    static void updateReplicationDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+                                                     BytesAndMessagesCount quota) {
+        resourceGroupDispatchLimiter.update(quota.messages, quota.bytes);
+    }
+
+    static ResourceGroupDispatchLimiter newDispatchRateLimiter(
+            org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup,
+            ScheduledExecutorService executorService) {
+        long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1);
+        long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L);
+        return new ResourceGroupDispatchLimiter(executorService, msgs, bytes);
+    }
+
+    static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+                                          org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup) {
+        long msgs = Optional.ofNullable(resourceGroup.getDispatchRateInMsgs()).orElse(-1);
+        long bytes = Optional.ofNullable(resourceGroup.getDispatchRateInBytes()).orElse(-1L);
+        resourceGroupDispatchLimiter.update(msgs, bytes);
+    }
+
+    static void updateDispatchRateLimiter(ResourceGroupDispatchLimiter resourceGroupDispatchLimiter,
+                                          BytesAndMessagesCount quota) {
+        resourceGroupDispatchLimiter.update(quota.messages, quota.bytes);
+    }
+}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
index 1f4b3f32cb9..c4a774aa28b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupService.java
@@ -19,14 +19,22 @@
 package org.apache.pulsar.broker.resourcegroup;
 
 import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.annotations.VisibleForTesting;
 import io.prometheus.client.Counter;
+import io.prometheus.client.Gauge;
 import io.prometheus.client.Summary;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
 import lombok.Getter;
 import lombok.val;
 import org.apache.pulsar.broker.PulsarService;
@@ -60,7 +68,7 @@ public class ResourceGroupService implements AutoCloseable{
     public ResourceGroupService(PulsarService pulsar) {
         this.pulsar = pulsar;
         this.timeUnitScale = TimeUnit.SECONDS;
-        this.quotaCalculator = new ResourceQuotaCalculatorImpl();
+        this.quotaCalculator = new ResourceQuotaCalculatorImpl(pulsar);
         this.resourceUsageTransportManagerMgr = pulsar.getResourceUsageTransportManager();
         this.rgConfigListener = new ResourceGroupConfigListener(this, pulsar);
         this.initialize();
@@ -167,9 +175,11 @@ public class ResourceGroupService implements AutoCloseable{
 
         long tenantRefCount = rg.getResourceGroupNumOfTenantRefs();
         long nsRefCount = rg.getResourceGroupNumOfNSRefs();
-        if ((tenantRefCount + nsRefCount) > 0) {
+        long topicRefCount = rg.getResourceGroupNumOfTopicRefs();
+        if ((tenantRefCount + nsRefCount + topicRefCount) > 0) {
             String errMesg = "Resource group " + name + " still has " + tenantRefCount + " tenant refs";
             errMesg += " and " + nsRefCount + " namespace refs on it";
+            errMesg += " and " + topicRefCount + " topic refs on it";
             throw new PulsarAdminException(errMesg);
         }
 
@@ -288,11 +298,92 @@ public class ResourceGroupService implements AutoCloseable{
             throw new PulsarAdminException(errMesg);
         }
 
+        aggregateLock.lock();
+
+        Set<String> invalidateAllKeyForProduce = new HashSet<>();
+        topicProduceStats.asMap().forEach((key, value) -> {
+            TopicName topicName = TopicName.get(key);
+            if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+                invalidateAllKeyForProduce.add(key);
+            }
+        });
+        topicProduceStats.invalidateAll(invalidateAllKeyForProduce);
+
+        Set<String> invalidateAllKeyForReplication = new HashSet<>();
+        topicToReplicatorsMap.forEach((key, value) -> {
+            TopicName topicName = TopicName.get(key);
+            if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+                topicToReplicatorsMap.remove(key);
+                value.forEach(n -> invalidateAllKeyForReplication.add(getReplicatorKey(topicName.toString(), n)));
+            }
+        });
+        replicationDispatchStats.invalidateAll(invalidateAllKeyForReplication);
+
+        Set<String> invalidateAllKeyForConsumer = new HashSet<>();
+        topicConsumeStats.asMap().forEach((key, value) -> {
+            TopicName topicName = TopicName.get(key);
+            if (topicName.getNamespaceObject().equals(fqNamespaceName)) {
+                invalidateAllKeyForConsumer.add(key);
+            }
+        });
+        topicConsumeStats.invalidateAll(invalidateAllKeyForConsumer);
+
+        aggregateLock.unlock();
         // Dissociate this NS-name from the RG.
         this.namespaceToRGsMap.remove(fqNamespaceName, rg);
         rgNamespaceUnRegisters.labels(resourceGroupName).inc();
     }
 
+    /**
+     * Registers a topic as a user of a resource group.
+     *
+     * @param resourceGroupName
+     * @param topicName         complete topic name
+     */
+    public void registerTopic(String resourceGroupName, TopicName topicName) {
+        ResourceGroup rg = resourceGroupsMap.get(resourceGroupName);
+        if (rg == null) {
+            throw new IllegalStateException("Resource group does not exist: " + resourceGroupName);
+        }
+
+        ResourceGroupOpStatus status = rg.registerUsage(topicName.toString(), ResourceGroupRefTypes.Topics,
+                true, this.resourceUsageTransportManagerMgr);
+        if (status == ResourceGroupOpStatus.Exists) {
+            String msg = String.format("Topic %s already references the target resource group %s",
+                    topicName, resourceGroupName);
+            throw new IllegalStateException(msg);
+        }
+
+        // Associate this topic-name with the RG.
+        this.topicToRGsMap.put(topicName, rg);
+        rgTopicRegisters.labels(resourceGroupName).inc();
+    }
+
+    /**
+     * UnRegisters a topic from a resource group.
+     *
+     * @param topicName complete topic name
+     */
+    public void unRegisterTopic(TopicName topicName) {
+        aggregateLock.lock();
+        String topicNameString = topicName.toString();
+        ResourceGroup remove = topicToRGsMap.remove(topicName);
+        if (remove != null) {
+            remove.registerUsage(topicNameString, ResourceGroupRefTypes.Topics,
+                    false, this.resourceUsageTransportManagerMgr);
+            rgTopicUnRegisters.labels(remove.resourceGroupName).inc();
+        }
+        topicProduceStats.invalidate(topicNameString);
+        topicConsumeStats.invalidate(topicNameString);
+        Set<String> replicators = topicToReplicatorsMap.remove(topicNameString);
+        if (replicators != null) {
+            List<String> keys = replicators.stream().map(n -> getReplicatorKey(topicNameString, n))
+                    .collect(Collectors.toList());
+            replicationDispatchStats.invalidateAll(keys);
+        }
+        aggregateLock.unlock();
+    }
+
     /**
      * Return the resource group associated with a namespace.
      *
@@ -303,6 +394,11 @@ public class ResourceGroupService implements AutoCloseable{
         return this.namespaceToRGsMap.get(namespaceName);
     }
 
+    @VisibleForTesting
+    public ResourceGroup getTopicResourceGroup(TopicName topicName) {
+        return this.topicToRGsMap.get(topicName);
+    }
+
     @Override
     public void close() throws Exception {
         if (aggregateLocalUsagePeriodicTask != null) {
@@ -314,12 +410,21 @@ public class ResourceGroupService implements AutoCloseable{
         resourceGroupsMap.clear();
         tenantToRGsMap.clear();
         namespaceToRGsMap.clear();
-        topicProduceStats.clear();
-        topicConsumeStats.clear();
+        topicProduceStats.invalidateAll();
+        topicConsumeStats.invalidateAll();
+        replicationDispatchStats.invalidateAll();
+    }
+
+    private void incrementUsage(ResourceGroup resourceGroup,
+                                ResourceGroupMonitoringClass monClass, BytesAndMessagesCount incStats)
+            throws PulsarAdminException {
+        resourceGroup.incrementLocalUsageStats(monClass, incStats);
+        rgLocalUsageBytes.labels(resourceGroup.resourceGroupName, monClass.name()).inc(incStats.bytes);
+        rgLocalUsageMessages.labels(resourceGroup.resourceGroupName, monClass.name()).inc(incStats.messages);
     }
 
     /**
-     * Increments usage stats for the resource groups associated with the given namespace and tenant.
+     * Increments usage stats for the resource groups associated with the given namespace, tenant, and topic.
      * Expected to be called when a message is produced or consumed on a topic, or when we calculate
      * usage periodically in the background by going through broker-service stats. [Not yet decided
      * which model we will follow.] Broker-service stats will be cumulative, while calls from the
@@ -327,22 +432,25 @@ public class ResourceGroupService implements AutoCloseable{
      *
      * If the tenant and NS are associated with different RGs, the statistics on both RGs are updated.
      * If the tenant and NS are associated with the same RG, the stats on the RG are updated only once
+     * If the tenant, NS and topic are associated with the same RG, the stats on the RG are updated only once
      * (to avoid a direct double-counting).
      * ToDo: will this distinction result in "expected semantics", or shock from users?
      * For now, the only caller is internal to this class.
      *
      * @param tenantName
-     * @param nsName
+     * @param nsName Complete namespace name
+     * @param topicName Complete topic name
      * @param monClass
      * @param incStats
      * @returns true if the stats were updated; false if nothing was updated.
      */
-    protected boolean incrementUsage(String tenantName, String nsName,
-                                  ResourceGroupMonitoringClass monClass,
-                                  BytesAndMessagesCount incStats) throws PulsarAdminException {
-        final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(tenantName, nsName));
+    protected boolean incrementUsage(String tenantName, String nsName, String topicName,
+                                     ResourceGroupMonitoringClass monClass,
+                                     BytesAndMessagesCount incStats) throws PulsarAdminException {
+        final ResourceGroup nsRG = this.namespaceToRGsMap.get(NamespaceName.get(nsName));
         final ResourceGroup tenantRG = this.tenantToRGsMap.get(tenantName);
-        if (tenantRG == null && nsRG == null) {
+        final ResourceGroup topicRG = this.topicToRGsMap.get(TopicName.get(topicName));
+        if (tenantRG == null && nsRG == null && topicRG == null) {
             return false;
         }
 
@@ -353,24 +461,40 @@ public class ResourceGroupService implements AutoCloseable{
             throw new PulsarAdminException(errMesg);
         }
 
-        if (nsRG == tenantRG) {
+        if (tenantRG == nsRG && nsRG == topicRG) {
             // Update only once in this case.
-            // Note that we will update both tenant and namespace RGs in other cases.
-            nsRG.incrementLocalUsageStats(monClass, incStats);
-            rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages);
-            rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+            // Note that we will update both tenant, namespace and topic RGs in other cases.
+            incrementUsage(tenantRG, monClass, incStats);
+            return true;
+        }
+
+        if (tenantRG != null && tenantRG == nsRG) {
+            // Tenant and Namespace GRs are same.
+            incrementUsage(tenantRG, monClass, incStats);
+            if (topicRG == null) {
+                return true;
+            }
+        }
+
+        if (nsRG != null && nsRG == topicRG) {
+            // Namespace and Topic GRs are same.
+            incrementUsage(nsRG, monClass, incStats);
             return true;
         }
 
         if (tenantRG != null) {
-            tenantRG.incrementLocalUsageStats(monClass, incStats);
-            rgLocalUsageMessages.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.messages);
-            rgLocalUsageBytes.labels(tenantRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+            // Tenant GR is different from other resource GR.
+            incrementUsage(tenantRG, monClass, incStats);
         }
+
         if (nsRG != null) {
-            nsRG.incrementLocalUsageStats(monClass, incStats);
-            rgLocalUsageMessages.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.messages);
-            rgLocalUsageBytes.labels(nsRG.resourceGroupName, monClass.name()).inc(incStats.bytes);
+            // Namespace GR is different from other resource GR.
+            incrementUsage(nsRG, monClass, incStats);
+        }
+
+        if (topicRG != null) {
+            // Topic GR is different from other resource GR.
+            incrementUsage(topicRG, monClass, incStats);
         }
 
         return true;
@@ -419,11 +543,17 @@ public class ResourceGroupService implements AutoCloseable{
         return rg;
     }
 
+    private String getReplicatorKey(String topic, String replicationRemoteCluster) {
+        return topic + replicationRemoteCluster;
+    }
+
     // Find the difference between the last time stats were updated for this topic, and the current
     // time. If the difference is positive, update the stats.
-    private void updateStatsWithDiff(String topicName, String tenantString, String nsString,
-                                     long accByteCount, long accMesgCount, ResourceGroupMonitoringClass monClass) {
-        ConcurrentHashMap<String, BytesAndMessagesCount> hm;
+    @VisibleForTesting
+    protected void updateStatsWithDiff(String topicName, String replicationRemoteCluster, String tenantString,
+                                       String nsString, long accByteCount, long accMsgCount,
+                                       ResourceGroupMonitoringClass monClass) {
+        Cache<String, BytesAndMessagesCount> hm;
         switch (monClass) {
             default:
                 log.error("updateStatsWithDiff: Unknown monitoring class={}; ignoring", monClass);
@@ -436,6 +566,10 @@ public class ResourceGroupService implements AutoCloseable{
             case Dispatch:
                 hm = this.topicConsumeStats;
                 break;
+
+            case ReplicationDispatch:
+                hm = this.replicationDispatchStats;
+                break;
         }
 
         BytesAndMessagesCount bmDiff = new BytesAndMessagesCount();
@@ -443,9 +577,22 @@ public class ResourceGroupService implements AutoCloseable{
         BytesAndMessagesCount bmNewCount = new BytesAndMessagesCount();
 
         bmNewCount.bytes = accByteCount;
-        bmNewCount.messages = accMesgCount;
-
-        bmOldCount = hm.get(topicName);
+        bmNewCount.messages = accMsgCount;
+
+        String key;
+        if (monClass == ResourceGroupMonitoringClass.ReplicationDispatch) {
+            key = getReplicatorKey(topicName, replicationRemoteCluster);
+            topicToReplicatorsMap.compute(topicName, (n, value) -> {
+                if (value == null) {
+                    value = new CopyOnWriteArraySet<>();
+                }
+                value.add(replicationRemoteCluster);
+                return value;
+            });
+        } else {
+            key = topicName;
+        }
+        bmOldCount = hm.getIfPresent(key);
         if (bmOldCount == null) {
             bmDiff.bytes = bmNewCount.bytes;
             bmDiff.messages = bmNewCount.messages;
@@ -459,14 +606,14 @@ public class ResourceGroupService implements AutoCloseable{
         }
 
         try {
-            boolean statsUpdated = this.incrementUsage(tenantString, nsString, monClass, bmDiff);
+            boolean statsUpdated = this.incrementUsage(tenantString, nsString, topicName, monClass, bmDiff);
             if (log.isDebugEnabled()) {
                 log.debug("updateStatsWithDiff for topic={}: monclass={} statsUpdated={} for tenant={}, namespace={}; "
                                 + "by {} bytes, {} mesgs",
                         topicName, monClass, statsUpdated, tenantString, nsString,
                         bmDiff.bytes, bmDiff.messages);
             }
-            hm.put(topicName, bmNewCount);
+            hm.put(key, bmNewCount);
         } catch (Throwable t) {
             log.error("updateStatsWithDiff: got ex={} while aggregating for {} side",
                     t.getMessage(), monClass);
@@ -488,11 +635,21 @@ public class ResourceGroupService implements AutoCloseable{
         return rgCalculatedQuotaBytes.labels(rgName, monClassName).get();
     }
 
+    // Visibility for testing.
+    protected static double getRgQuotaByte (String rgName, String monClassName) {
+        return rgCalculatedQuotaBytesGauge.labels(rgName, monClassName).get();
+    }
+
     // Visibility for testing.
     protected static double getRgQuotaMessageCount (String rgName, String monClassName) {
         return rgCalculatedQuotaMessages.labels(rgName, monClassName).get();
     }
 
+    // Visibility for testing.
+    protected static double getRgQuotaMessage(String rgName, String monClassName) {
+        return rgCalculatedQuotaMessagesGauge.labels(rgName, monClassName).get();
+    }
+
     // Visibility for testing.
     protected static double getRgLocalUsageByteCount (String rgName, String monClassName) {
         return rgLocalUsageBytes.labels(rgName, monClassName).get();
@@ -545,30 +702,40 @@ public class ResourceGroupService implements AutoCloseable{
         BrokerService bs = this.pulsar.getBrokerService();
         Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
 
+        aggregateLock.lock();
         for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
             final String topicName = entry.getKey();
             final TopicStats topicStats = entry.getValue();
             final TopicName topic = TopicName.get(topicName);
             final String tenantString = topic.getTenant();
-            final String nsString = topic.getNamespacePortion();
+            final String nsString = topic.getNamespace();
             final NamespaceName fqNamespace = topic.getNamespaceObject();
 
             // Can't use containsKey here, as that checks for exact equality
             // (we need a check for string-comparison).
             val tenantRG = this.tenantToRGsMap.get(tenantString);
             val namespaceRG = this.namespaceToRGsMap.get(fqNamespace);
-            if (tenantRG == null && namespaceRG == null) {
+            val topicRG = this.topicToRGsMap.get(topic);
+            if (tenantRG == null && namespaceRG == null && topicRG == null) {
                 // This topic's NS/tenant are not registered to any RG.
                 continue;
             }
 
-            this.updateStatsWithDiff(topicName, tenantString, nsString,
+            topicStats.getReplication().forEach((remoteCluster, stats) -> {
+                this.updateStatsWithDiff(topicName, remoteCluster, tenantString, nsString,
+                        stats.getBytesOutCounter(),
+                        stats.getMsgOutCounter(),
+                        ResourceGroupMonitoringClass.ReplicationDispatch
+                );
+            });
+            this.updateStatsWithDiff(topicName, null, tenantString, nsString,
                     topicStats.getBytesInCounter(), topicStats.getMsgInCounter(),
                     ResourceGroupMonitoringClass.Publish);
-            this.updateStatsWithDiff(topicName, tenantString, nsString,
+            this.updateStatsWithDiff(topicName, null, tenantString, nsString,
                     topicStats.getBytesOutCounter(), topicStats.getMsgOutCounter(),
                     ResourceGroupMonitoringClass.Dispatch);
         }
+        aggregateLock.unlock();
         double diffTimeSeconds = aggrUsageTimer.observeDuration();
         if (log.isDebugEnabled()) {
             log.debug("aggregateResourceGroupLocalUsages took {} milliseconds", diffTimeSeconds * 1000);
@@ -629,13 +796,7 @@ public class ResourceGroupService implements AutoCloseable{
                             globUsageMessagesArray);
 
                     BytesAndMessagesCount oldBMCount = resourceGroup.updateLocalQuota(monClass, updatedQuota);
-                    // Guard against unconfigured quota settings, for which computeLocalQuota will return negative.
-                    if (updatedQuota.messages >= 0) {
-                        rgCalculatedQuotaMessages.labels(rgName, monClass.name()).inc(updatedQuota.messages);
-                    }
-                    if (updatedQuota.bytes >= 0) {
-                        rgCalculatedQuotaBytes.labels(rgName, monClass.name()).inc(updatedQuota.bytes);
-                    }
+                    incRgCalculatedQuota(rgName, monClass, updatedQuota, resourceUsagePublishPeriodInSeconds);
                     if (oldBMCount != null) {
                         long messagesIncrement = updatedQuota.messages - oldBMCount.messages;
                         long bytesIncrement = updatedQuota.bytes - oldBMCount.bytes;
@@ -686,8 +847,6 @@ public class ResourceGroupService implements AutoCloseable{
                         newPeriodInSeconds,
                         timeUnitScale);
             this.resourceUsagePublishPeriodInSeconds = newPeriodInSeconds;
-            maxIntervalForSuppressingReportsMSecs =
-                    TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds;
         }
     }
 
@@ -705,9 +864,11 @@ public class ResourceGroupService implements AutoCloseable{
                     periodInSecs,
                     periodInSecs,
                     this.timeUnitScale);
-        maxIntervalForSuppressingReportsMSecs =
-                TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds) * MaxUsageReportSuppressRounds;
-
+        long resourceUsagePublishPeriodInMS = TimeUnit.SECONDS.toMillis(this.resourceUsagePublishPeriodInSeconds);
+        long statsCacheInMS = resourceUsagePublishPeriodInMS * 2;
+        topicProduceStats = newStatsCache(statsCacheInMS);
+        topicConsumeStats = newStatsCache(statsCacheInMS);
+        replicationDispatchStats = newStatsCache(statsCacheInMS);
     }
 
     private void checkRGCreateParams(String rgName, org.apache.pulsar.common.policies.data.ResourceGroup rgConfig)
@@ -726,6 +887,28 @@ public class ResourceGroupService implements AutoCloseable{
         }
     }
 
+    static void incRgCalculatedQuota(String rgName, ResourceGroupMonitoringClass monClass,
+                                     BytesAndMessagesCount updatedQuota, long resourceUsagePublishPeriodInSeconds) {
+        // Guard against unconfigured quota settings, for which computeLocalQuota will return negative.
+        if (updatedQuota.messages >= 0) {
+            rgCalculatedQuotaMessages.labels(rgName, monClass.name())
+                    .inc(updatedQuota.messages * resourceUsagePublishPeriodInSeconds);
+            rgCalculatedQuotaMessagesGauge.labels(rgName, monClass.name()).set(updatedQuota.messages);
+        }
+        if (updatedQuota.bytes >= 0) {
+            rgCalculatedQuotaBytes.labels(rgName, monClass.name())
+                    .inc(updatedQuota.bytes * resourceUsagePublishPeriodInSeconds);
+            rgCalculatedQuotaBytesGauge.labels(rgName, monClass.name()).set(updatedQuota.bytes);
+        }
+    }
+
+    @VisibleForTesting
+    protected Cache<String, BytesAndMessagesCount> newStatsCache(long durationMS) {
+        return Caffeine.newBuilder()
+                .expireAfterAccess(durationMS, TimeUnit.MILLISECONDS)
+                .build();
+    }
+
     private static final Logger log = LoggerFactory.getLogger(ResourceGroupService.class);
 
     @Getter
@@ -745,11 +928,14 @@ public class ResourceGroupService implements AutoCloseable{
 
     // Given a qualified NS-name (i.e., in "tenant/namespace" format), record its associated resource-group
     private ConcurrentHashMap<NamespaceName, ResourceGroup> namespaceToRGsMap = new ConcurrentHashMap<>();
+    private ConcurrentHashMap<TopicName, ResourceGroup> topicToRGsMap = new ConcurrentHashMap<>();
+    private ConcurrentHashMap<String, Set<String>> topicToReplicatorsMap = new ConcurrentHashMap<>();
+    private ReentrantLock aggregateLock = new ReentrantLock();
 
     // Maps to maintain the usage per topic, in produce/consume directions.
-    private ConcurrentHashMap<String, BytesAndMessagesCount> topicProduceStats = new ConcurrentHashMap<>();
-    private ConcurrentHashMap<String, BytesAndMessagesCount> topicConsumeStats = new ConcurrentHashMap<>();
-
+    private Cache<String, BytesAndMessagesCount> topicProduceStats;
+    private Cache<String, BytesAndMessagesCount> topicConsumeStats;
+    private Cache<String, BytesAndMessagesCount> replicationDispatchStats;
 
     // The task that periodically re-calculates the quota budget for local usage.
     private ScheduledFuture<?> aggregateLocalUsagePeriodicTask;
@@ -762,22 +948,6 @@ public class ResourceGroupService implements AutoCloseable{
     // Allow a pluggable scale on time units; for testing periodic functionality.
     private TimeUnit timeUnitScale;
 
-    // The maximum number of successive rounds that we can suppress reporting local usage, because there was no
-    // substantial change from the prior round. This is to ensure the reporting does not become too chatty.
-    // Set this value to one more than the cadence of sending reports; e.g., if you want to send every 3rd report,
-    // set the value to 4.
-    // Setting this to 0 will make us report in every round.
-    // Don't set to negative values; behavior will be "undefined".
-    protected static final int MaxUsageReportSuppressRounds = 5;
-
-    // Convenient shorthand, for MaxUsageReportSuppressRounds converted to a time interval in milliseconds.
-    protected static long maxIntervalForSuppressingReportsMSecs;
-
-    // The percentage difference that is considered "within limits" to suppress usage reporting.
-    // Setting this to 0 will also make us report in every round.
-    // Don't set it to negative values; behavior will be "undefined".
-    protected static final float UsageReportSuppressionTolerancePercentage = 5;
-
     // Labels for the various counters used here.
     private static final String[] resourceGroupLabel = {"ResourceGroup"};
     private static final String[] resourceGroupMonitoringclassLabels = {"ResourceGroup", "MonitoringClass"};
@@ -787,11 +957,21 @@ public class ResourceGroupService implements AutoCloseable{
             .help("Bytes quota calculated for resource group")
             .labelNames(resourceGroupMonitoringclassLabels)
             .register();
+    private static final Gauge rgCalculatedQuotaBytesGauge = Gauge.build()
+            .name("pulsar_resource_group_calculated_bytes_quota_gauge")
+            .help("Bytes quota calculated for resource group")
+            .labelNames(resourceGroupMonitoringclassLabels)
+            .register();
     private static final Counter rgCalculatedQuotaMessages = Counter.build()
             .name("pulsar_resource_group_calculated_messages_quota")
             .help("Messages quota calculated for resource group")
             .labelNames(resourceGroupMonitoringclassLabels)
             .register();
+    private static final Gauge rgCalculatedQuotaMessagesGauge = Gauge.build()
+            .name("pulsar_resource_group_calculated_messages_quota_gauge")
+            .help("Messages quota calculated for resource group")
+            .labelNames(resourceGroupMonitoringclassLabels)
+            .register();
 
     private static final Counter rgLocalUsageBytes = Counter.build()
             .name("pulsar_resource_group_bytes_used")
@@ -832,6 +1012,17 @@ public class ResourceGroupService implements AutoCloseable{
             .labelNames(resourceGroupLabel)
             .register();
 
+    private static final Counter rgTopicRegisters = Counter.build()
+            .name("pulsar_resource_group_topic_registers")
+            .help("Number of registrations of topics")
+            .labelNames(resourceGroupLabel)
+            .register();
+    private static final Counter rgTopicUnRegisters = Counter.build()
+            .name("pulsar_resource_group_topic_unregisters")
+            .help("Number of un-registrations of topics")
+            .labelNames(resourceGroupLabel)
+            .register();
+
     private static final Summary rgUsageAggregationLatency = Summary.build()
             .quantile(0.5, 0.05)
             .quantile(0.9, 0.01)
@@ -847,15 +1038,25 @@ public class ResourceGroupService implements AutoCloseable{
             .register();
 
     @VisibleForTesting
-    ConcurrentHashMap getTopicConsumeStats() {
+    Cache<String, BytesAndMessagesCount> getTopicConsumeStats() {
         return this.topicConsumeStats;
     }
 
     @VisibleForTesting
-    ConcurrentHashMap getTopicProduceStats() {
+    Cache<String, BytesAndMessagesCount> getTopicProduceStats() {
         return this.topicProduceStats;
     }
 
+    @VisibleForTesting
+    Cache<String, BytesAndMessagesCount> getReplicationDispatchStats() {
+        return this.replicationDispatchStats;
+    }
+
+    @VisibleForTesting
+    ConcurrentHashMap<String, Set<String>> getTopicToReplicatorsMap() {
+        return this.topicToReplicatorsMap;
+    }
+
     @VisibleForTesting
     ScheduledFuture<?> getAggregateLocalUsagePeriodicTask() {
         return this.aggregateLocalUsagePeriodicTask;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
index 41291100a0d..fe1808077d7 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImpl.java
@@ -18,14 +18,22 @@
  */
 package org.apache.pulsar.broker.resourcegroup;
 
-import static java.lang.Float.max;
 import static java.lang.Math.abs;
+import java.util.concurrent.TimeUnit;
 import lombok.val;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
+    private final PulsarService pulsarService;
+
+    public ResourceQuotaCalculatorImpl(PulsarService pulsarService) {
+        this.pulsarService = pulsarService;
+    }
+
     @Override
     public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) throws PulsarAdminException {
         // ToDo: work out the initial conditions: we may allow a small number of "first few iterations" to go
@@ -53,13 +61,14 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
             throw new PulsarAdminException(errMesg);
         }
 
-        // If the total usage is zero (which may happen during initial transients), just return the configured value.
+        // If the total/myUsage usage is zero (which may happen during initial transients, or when there is no
+        // traffic in the current broker), just return the configured value.
         // The caller is expected to check the value returned, or not call here with a zero global usage.
         // [This avoids a division by zero when calculating the local share.]
-        if (totalUsage == 0) {
+        if (myUsage == 0 || totalUsage == 0) {
             if (log.isDebugEnabled()) {
-                log.debug("computeLocalQuota: totalUsage is zero; "
-                        + "returning the configured usage ({}) as new local quota",
+                log.debug("computeLocalQuota: totalUsage or myUsage is zero; "
+                                + "returning the configured usage ({}) as new local quota",
                         confUsage);
             }
             return confUsage;
@@ -72,16 +81,25 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
             log.warn(errMesg);
         }
 
-        // How much unused capacity is left over?
-        float residual = confUsage - totalUsage;
+        int resourceUsageTransportPublishIntervalInSecs =
+                pulsarService.getConfiguration().getResourceUsageTransportPublishIntervalInSecs();
+        double resourceGroupLocalQuotaThreshold =
+                pulsarService.getConfiguration().getResourceGroupLocalQuotaThreshold();
+        float totalRate = (float) totalUsage / resourceUsageTransportPublishIntervalInSecs;
+        double adjustedConfUsage = confUsage * resourceGroupLocalQuotaThreshold;
+        if (totalRate <= adjustedConfUsage) {
+            log.info("computeLocalQuota: total usage ({}) is less than the "
+                            + "configured usage * threshold ({}), "
+                            + "returning the configured usage ({}) as new local quota",
+                    totalRate, adjustedConfUsage, confUsage);
+            return confUsage;
+        }
 
         // New quota is the old usage incremented by any residual as a ratio of the local usage to the total usage.
         // This should result in the calculatedQuota increasing proportionately if total usage is less than the
         // configured usage, and reducing proportionately if the total usage is greater than the configured usage.
-        // Capped to 1, to prevent negative or zero setting of quota.
-        // the rate limiter code assumes that rate value of 0 or less to mean that no rate limit should be applied
         float myUsageFraction = (float) myUsage / totalUsage;
-        float calculatedQuota = max(myUsage + residual * myUsageFraction, 1);
+        float calculatedQuota = Math.max(myUsageFraction * confUsage, 1);
 
         val longCalculatedQuota = (long) calculatedQuota;
         log.info("computeLocalQuota: myUsage={}, totalUsage={}, myFraction={}; newQuota returned={} [long: {}]",
@@ -93,34 +111,50 @@ public class ResourceQuotaCalculatorImpl implements ResourceQuotaCalculator {
     @Override
     // Return true if a report needs to be sent for the current round; false if it can be suppressed for this round.
     public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes,
-                                                    long currentMessagesUsed, long lastReportedMessages,
-                                                    long lastReportTimeMSecsSinceEpoch) {
-        // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report.
-        long currentTimeMSecs = System.currentTimeMillis();
-        long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch;
-        if (mSecsSinceLastReport >= ResourceGroupService.maxIntervalForSuppressingReportsMSecs) {
+                                          long currentMessagesUsed, long lastReportedMessages,
+                                          long lastReportTimeMSecsSinceEpoch) {
+        ServiceConfiguration configuration = pulsarService.getConfiguration();
+        long resourceUsageMaxUsageReportSuppressRounds = configuration.getResourceUsageMaxUsageReportSuppressRounds();
+        if (resourceUsageMaxUsageReportSuppressRounds == 0) {
             return true;
+        } else if (resourceUsageMaxUsageReportSuppressRounds > 0) {
+            // If we are about to go more than maxUsageReportSuppressRounds without reporting, send a report.
+            long currentTimeMSecs = System.currentTimeMillis();
+            long mSecsSinceLastReport = currentTimeMSecs - lastReportTimeMSecsSinceEpoch;
+            if (mSecsSinceLastReport >= TimeUnit.SECONDS.toMillis(
+                    (long) configuration.getResourceUsageTransportPublishIntervalInSecs()
+                            * configuration.getResourceUsageMaxUsageReportSuppressRounds())) {
+                return true;
+            }
         }
 
         // If the percentage change (increase or decrease) in usage is more than a threshold for
         // either bytes or messages, send a report.
-        final float toleratedDriftPercentage = ResourceGroupService.UsageReportSuppressionTolerancePercentage;
-        if (currentBytesUsed > 0) {
-            long diff = abs(currentBytesUsed - lastReportedBytes);
-            float diffPercentage = (float) diff * 100 / lastReportedBytes;
-            if (diffPercentage > toleratedDriftPercentage) {
-                return true;
-            }
+        final float toleratedDriftPercentage = configuration.getResourceUsageReportSuppressionTolerancePercentage();
+
+        if (needToReportLocalUsage(currentBytesUsed, lastReportedBytes, toleratedDriftPercentage)) {
+            return true;
         }
 
-        if (currentMessagesUsed > 0) {
-            long diff = abs(currentMessagesUsed - lastReportedMessages);
-            float diffPercentage = (float) diff * 100 / lastReportedMessages;
-            if (diffPercentage > toleratedDriftPercentage) {
+        if (needToReportLocalUsage(currentMessagesUsed, lastReportedMessages, toleratedDriftPercentage)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private boolean needToReportLocalUsage(long currentUsed, long lastReported, float toleratedDriftPercentage) {
+        if (currentUsed > 0) {
+            if (lastReported == 0) {
                 return true;
             }
+            if (toleratedDriftPercentage <= 0) {
+                return true;
+            }
+            long diff = abs(currentUsed - lastReported);
+            float diffPercentage = (float) diff * 100 / lastReported;
+            return diffPercentage > toleratedDriftPercentage;
         }
-
         return false;
     }
 
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 7ca210280a7..2fc8fe883c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -51,7 +51,9 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.PulsarService;
 import org.apache.pulsar.broker.ServiceConfiguration;
 import org.apache.pulsar.broker.resourcegroup.ResourceGroup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.resourcegroup.ResourceGroupPublishLimiter;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService;
 import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException;
 import org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException;
 import org.apache.pulsar.broker.service.BrokerServiceException.ProducerFencedException;
@@ -123,6 +125,9 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
 
     protected volatile ResourceGroupPublishLimiter resourceGroupPublishLimiter;
 
+    @Getter
+    protected volatile Optional<ResourceGroupDispatchLimiter> resourceGroupDispatchRateLimiter = Optional.empty();
+
     protected boolean preciseTopicPublishRateLimitingEnable;
 
     @Getter
@@ -251,6 +256,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
         topicPolicies.getSchemaValidationEnforced().updateTopicValue(data.getSchemaValidationEnforced());
         topicPolicies.getEntryFilters().updateTopicValue(data.getEntryFilters());
         this.subscriptionPolicies = data.getSubscriptionPolicies();
+        topicPolicies.getResourceGroupName().updateTopicValue(data.getResourceGroupName());
 
         updateEntryFilters();
     }
@@ -302,6 +308,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
         updateNamespaceDispatchRate(namespacePolicies, brokerService.getPulsar().getConfig().getClusterName());
         topicPolicies.getSchemaValidationEnforced().updateNamespaceValue(namespacePolicies.schema_validation_enforced);
         topicPolicies.getEntryFilters().updateNamespaceValue(namespacePolicies.entryFilters);
+        topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name);
 
         updateEntryFilters();
     }
@@ -1175,29 +1182,53 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener<TopicP
 
     public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) {
         requireNonNull(namespacePolicies);
-        // attach the resource-group level rate limiters, if set
-        String rgName = namespacePolicies.resource_group_name;
+        topicPolicies.getResourceGroupName().updateNamespaceValue(namespacePolicies.resource_group_name);
+        updateResourceGroupLimiter();
+    }
+
+    public void updateResourceGroupLimiter() {
+        String rgName = topicPolicies.getResourceGroupName().get();
         if (rgName != null) {
-            final ResourceGroup resourceGroup =
-                    brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName);
+            ResourceGroupService resourceGroupService = brokerService.getPulsar().getResourceGroupServiceManager();
+            final ResourceGroup resourceGroup = resourceGroupService.resourceGroupGet(rgName);
             if (resourceGroup != null) {
+                TopicName topicName = TopicName.get(topic);
+                resourceGroupService.unRegisterTopic(topicName);
+                String topicRg = topicPolicies.getResourceGroupName().getTopicValue();
+                if (topicRg != null) {
+                    try {
+                        resourceGroupService.registerTopic(topicRg, topicName);
+                    } catch (Exception e) {
+                        log.error("Failed to register resource group {} for topic {}", rgName, topic);
+                        return;
+                    }
+                }
                 this.resourceGroupRateLimitingEnabled = true;
                 this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter();
                 this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), this::enableCnxAutoRead);
+                this.resourceGroupDispatchRateLimiter = Optional.of(resourceGroup.getResourceGroupDispatchLimiter());
                 log.info("Using resource group {} rate limiter for topic {}", rgName, topic);
-                return;
             }
         } else {
-            if (this.resourceGroupRateLimitingEnabled) {
-                this.resourceGroupPublishLimiter.unregisterRateLimitFunction(this.getName());
-                this.resourceGroupPublishLimiter = null;
-                this.resourceGroupRateLimitingEnabled = false;
-            }
+            closeResourceGroupLimiter();
+
             /* Namespace detached from resource group. Enable the producer read */
             enableProducerReadForPublishRateLimiting();
         }
     }
 
+    protected void closeResourceGroupLimiter() {
+        if (resourceGroupRateLimitingEnabled) {
+            this.resourceGroupPublishLimiter = null;
+            this.resourceGroupDispatchRateLimiter = Optional.empty();
+            this.resourceGroupRateLimitingEnabled = false;
+        }
+        ResourceGroupService resourceGroupServiceManager = brokerService.getPulsar().getResourceGroupServiceManager();
+        if (resourceGroupServiceManager != null) {
+            resourceGroupServiceManager.unRegisterTopic(TopicName.get(topic));
+        }
+    }
+
     public void updateEntryFilters() {
         if (isSystemTopic()) {
             entryFilters = Pair.of(null, Collections.emptyList());
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
index 5c314397da8..3f821adda11 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.broker.service;
 
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter;
 import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
 
@@ -48,6 +49,10 @@ public interface Replicator {
         return Optional.empty();
     }
 
+    default Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+        return Optional.empty();
+    }
+
     boolean isConnected();
 
     long getNumberOfEntriesInBacklog();
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
index 2e961b793d8..6e7fef67cd9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java
@@ -25,6 +25,7 @@ import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import org.apache.bookkeeper.mledger.Position;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter;
 import org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter;
 import org.apache.pulsar.broker.service.plugin.EntryFilter;
@@ -339,6 +340,10 @@ public interface Topic {
         return Optional.empty();
     }
 
+    default Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+        return Optional.empty();
+    }
+
     default Optional<SubscribeRateLimiter> getSubscribeRateLimiter() {
         return Optional.empty();
     }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index 0a725183bde..250fbbe1e8a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -181,7 +181,7 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
                         isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema;
                     }
                     updatePublishRateLimiter();
-                    updateResourceGroupLimiter(policies);
+                    updateResourceGroupLimiter();
                     return updateClusterMigrated();
                 });
     }
@@ -457,6 +457,9 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
                             brokerService.executor().execute(() -> {
                                 brokerService.removeTopicFromCache(NonPersistentTopic.this);
                                 unregisterTopicPolicyListener();
+
+                                closeResourceGroupLimiter();
+
                                 log.info("[{}] Topic deleted", topic);
                                 deleteFuture.complete(null);
                             });
@@ -535,6 +538,7 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol
             brokerService.executor().execute(() -> {
                 brokerService.removeTopicFromCache(NonPersistentTopic.this);
                 unregisterTopicPolicyListener();
+                closeResourceGroupLimiter();
                 closeFuture.complete(null);
             });
         }).exceptionally(exception -> {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
index 3390c3a2885..e5a81fc76f6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java
@@ -26,6 +26,7 @@ import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.transaction.TxnID;
@@ -81,8 +82,8 @@ public class GeoPersistentReplicator extends PersistentReplicator {
     }
 
     @Override
-    protected boolean replicateEntries(List<Entry> entries) {
-        boolean atLeastOneMessageSentForReplication = false;
+    protected ReplicationStatus replicateEntries(List<Entry> entries) {
+        ReplicationStatus replicationStatus = ReplicationStatus.NO_ENTRIES_REPLICATED;
         boolean isEnableReplicatedSubscriptions =
                 brokerService.pulsar().getConfiguration().isEnableReplicatedSubscriptions();
 
@@ -165,7 +166,22 @@ public class GeoPersistentReplicator extends PersistentReplicator {
                     continue;
                 }
 
-                dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength()));
+                int msgCount = msg.getMessageBuilder().hasNumMessagesInBatch()
+                        ? msg.getMessageBuilder().getNumMessagesInBatch() : 1;
+                dispatchRateLimiter.ifPresent(
+                        rateLimiter -> rateLimiter.tryDispatchPermit(msgCount, entry.getLength()));
+
+                ResourceGroupDispatchLimiter resourceGroupLimiter = resourceGroupDispatchRateLimiter.orElse(null);
+                if (resourceGroupLimiter != null) {
+                    if (!resourceGroupLimiter.tryAcquire(msgCount, entry.getLength())) {
+                        entry.release();
+                        msg.recycle();
+                        cursor.cancelPendingReadRequest();
+                        cursor.rewind();
+                        return ReplicationStatus.RATE_LIMITED;
+                    }
+                }
+
                 msg.setReplicatedFrom(localCluster);
 
                 headersAndPayload.retain();
@@ -196,15 +212,17 @@ public class GeoPersistentReplicator extends PersistentReplicator {
                     msg.getMessageBuilder().clearTxnidMostBits();
                     msg.getMessageBuilder().clearTxnidLeastBits();
                     msgOut.recordEvent(headersAndPayload.readableBytes());
+                    msgOutCounter.add(msgCount);
+                    bytesOutCounter.add(length);
                     // Increment pending messages for messages produced locally
                     PENDING_MESSAGES_UPDATER.incrementAndGet(this);
                     producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
-                    atLeastOneMessageSentForReplication = true;
+                    replicationStatus = ReplicationStatus.AT_LEAST_ONE_REPLICATED;
                 }
             }
         } catch (Exception e) {
             log.error("[{}] Unexpected exception in replication task: {}", replicatorId, e.getMessage(), e);
         }
-        return atLeastOneMessageSentForReplication;
+        return replicationStatus;
     }
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
index 9e8483be1a7..8b706cfe7d1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java
@@ -52,6 +52,7 @@ import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory;
 import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker;
 import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers;
 import org.apache.pulsar.broker.service.BrokerServiceException;
 import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException;
@@ -481,6 +482,27 @@ public class PersistentDispatcherMultipleConsumers extends AbstractDispatcherMul
                     bytesToRead = calculateToRead.getRight();
                 }
             }
+
+            if (topic.getResourceGroupDispatchRateLimiter().isPresent()) {
+                ResourceGroupDispatchLimiter limiter = topic.getResourceGroupDispatchRateLimiter().get();
+                long availableDispatchRateLimitOnMsg = limiter.getAvailableDispatchRateLimitOnMsg();
+                long availableDispatchRateLimitOnByte = limiter.getAvailableDispatchRateLimitOnByte();
+                if (availableDispatchRateLimitOnMsg == 0 || availableDispatchRateLimitOnByte == 0) {
+                    if (log.isDebugEnabled()) {
+                        log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{}, schedule after a {}",
+                                name, limiter.getDispatchRateOnMsg(), limiter.getDispatchRateOnByte(),
+                                MESSAGE_RATE_BACKOFF_MS);
+                    }
+                    reScheduleRead();
+                    return Pair.of(-1, -1L);
+                } else {
+                    Pair<Integer, Long> calculateToRead =
+                            computeReadLimits(messagesToRead, (int) availableDispatchRateLimitOnMsg, bytesToRead,
+                                    availableDispatchRateLimitOnByte);
+                    messagesToRead = calculateToRead.getLeft();
+                    bytesToRead = calculateToRead.getRight();
+                }
+            }
         }
 
         if (havePendingReplayRead) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
index a138fde7a72..575be9eaeb5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java
@@ -40,6 +40,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadE
 import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer;
 import org.apache.pulsar.broker.service.Consumer;
 import org.apache.pulsar.broker.service.Dispatcher;
@@ -462,6 +463,30 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher
                     bytesToRead = calculateToRead.getRight();
                 }
             }
+
+            if (topic.getResourceGroupDispatchRateLimiter().isPresent()) {
+                ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = topic.getResourceGroupDispatchRateLimiter()
+                        .get();
+                long availableDispatchRateLimitOnMsg =
+                        resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg();
+                long availableDispatchRateLimitOnByte =
+                        resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte();
+                if (availableDispatchRateLimitOnMsg == 0 || availableDispatchRateLimitOnByte == 0) {
+                    if (log.isDebugEnabled()) {
+                        log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{}, schedule after a {}",
+                                name, availableDispatchRateLimitOnMsg, availableDispatchRateLimitOnByte,
+                                MESSAGE_RATE_BACKOFF_MS);
+                    }
+                    reScheduleRead();
+                    return Pair.of(-1, -1L);
+                } else {
+                    Pair<Integer, Long> calculateToRead =
+                            computeReadLimits(messagesToRead, (int) availableDispatchRateLimitOnMsg, bytesToRead,
+                                    availableDispatchRateLimitOnByte);
+                    messagesToRead = calculateToRead.getLeft();
+                    bytesToRead = calculateToRead.getRight();
+                }
+            }
         }
 
         // If messagesToRead is 0 or less, correct it to 1 to prevent IllegalArgumentException
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
index a0f58e0874f..dd6c6b44cfa 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java
@@ -33,6 +33,9 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.LongAdder;
+import lombok.AllArgsConstructor;
+import lombok.Data;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
@@ -48,6 +51,9 @@ import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.impl.PositionImpl;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService;
 import org.apache.pulsar.broker.service.AbstractReplicator;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.Replicator;
@@ -60,6 +66,7 @@ import org.apache.pulsar.client.impl.ProducerImpl;
 import org.apache.pulsar.client.impl.PulsarClientImpl;
 import org.apache.pulsar.client.impl.SendCallback;
 import org.apache.pulsar.common.api.proto.MarkerType;
+import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies;
 import org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.stats.Rate;
@@ -75,6 +82,7 @@ public abstract class PersistentReplicator extends AbstractReplicator
     protected final ManagedCursor cursor;
 
     protected Optional<DispatchRateLimiter> dispatchRateLimiter = Optional.empty();
+    protected Optional<ResourceGroupDispatchLimiter> resourceGroupDispatchRateLimiter = Optional.empty();
     private final Object dispatchRateLimiterLock = new Object();
 
     private int readBatchSize;
@@ -97,6 +105,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
 
     protected final Rate msgOut = new Rate();
     protected final Rate msgExpired = new Rate();
+    protected final LongAdder bytesOutCounter = new LongAdder();
+    protected final LongAdder msgOutCounter = new LongAdder();
 
     protected int messageTTLInSeconds = 0;
 
@@ -112,8 +122,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
     protected volatile boolean fetchSchemaInProgress = false;
 
     public PersistentReplicator(String localCluster, PersistentTopic localTopic, ManagedCursor cursor,
-                                   String remoteCluster, String remoteTopic,
-                                   BrokerService brokerService, PulsarClientImpl replicationClient)
+                                String remoteCluster, String remoteTopic,
+                                BrokerService brokerService, PulsarClientImpl replicationClient)
             throws PulsarServerException {
         super(localCluster, localTopic, remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(),
                 brokerService, replicationClient);
@@ -200,15 +210,32 @@ public abstract class PersistentReplicator extends AbstractReplicator
         }
     }
 
+
+    @Data
+    @AllArgsConstructor
+    private static class AvailablePermits {
+        private int messages;
+        private long bytes;
+
+        /**
+         * messages, bytes
+         * 0, O:  Producer queue is full, no permits.
+         * -1, -1:  Rate Limiter reaches limit.
+         * >0, >0:  available permits for read entries.
+         */
+        public boolean isExceeded() {
+            return messages == -1 && bytes == -1;
+        }
+
+        public boolean isReadable() {
+            return messages > 0 && bytes > 0;
+        }
+    }
+
     /**
      * Calculate available permits for read entries.
-     *
-     * @return
-     *   0:  Producer queue is full, no permits.
-     *  -1:  Rate Limiter reaches limit.
-     *  >0:  available permits for read entries.
      */
-    private int getAvailablePermits() {
+    private AvailablePermits getAvailablePermits() {
         int availablePermits = producerQueueSize - PENDING_MESSAGES_UPDATER.get(this);
 
         // return 0, if Producer queue is full, it will pause read entries.
@@ -217,14 +244,18 @@ public abstract class PersistentReplicator extends AbstractReplicator
                 log.debug("[{}] Producer queue is full, availablePermits: {}, pause reading",
                         replicatorId, availablePermits);
             }
-            return 0;
+            return new AvailablePermits(0, 0);
         }
 
+        long availablePermitsOnMsg = -1;
+        long availablePermitsOnByte = -1;
+
         // handle rate limit
         if (dispatchRateLimiter.isPresent() && dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) {
             DispatchRateLimiter rateLimiter = dispatchRateLimiter.get();
-            // no permits from rate limit
-            if (!rateLimiter.hasMessageDispatchPermit()) {
+            availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
+            availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte();
+            if (availablePermitsOnByte == 0 || availablePermitsOnMsg == 0) {
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] message-read exceeded topic replicator message-rate {}/{},"
                                     + " schedule after a {}",
@@ -233,17 +264,45 @@ public abstract class PersistentReplicator extends AbstractReplicator
                             rateLimiter.getDispatchRateOnByte(),
                             MESSAGE_RATE_BACKOFF_MS);
                 }
-                return -1;
+                return new AvailablePermits(-1, -1);
             }
+        }
 
-            // if dispatch-rate is in msg then read only msg according to available permit
-            long availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
-            if (availablePermitsOnMsg > 0) {
-                availablePermits = Math.min(availablePermits, (int) availablePermitsOnMsg);
+        if (resourceGroupDispatchRateLimiter.isPresent()) {
+            ResourceGroupDispatchLimiter rateLimiter = resourceGroupDispatchRateLimiter.get();
+            long rgAvailablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg();
+            long rgAvailablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte();
+            if (rgAvailablePermitsOnMsg == 0 || rgAvailablePermitsOnByte == 0) {
+                if (log.isDebugEnabled()) {
+                    log.debug("[{}] message-read exceeded resourcegroup message-rate {}/{},"
+                                    + " schedule after a {}",
+                            replicatorId,
+                            rateLimiter.getDispatchRateOnMsg(),
+                            rateLimiter.getDispatchRateOnByte(),
+                            MESSAGE_RATE_BACKOFF_MS);
+                }
+                return new AvailablePermits(-1, -1);
+            }
+            if (availablePermitsOnMsg == -1) {
+                availablePermitsOnMsg = rgAvailablePermitsOnMsg;
+            } else {
+                availablePermitsOnMsg = Math.min(rgAvailablePermitsOnMsg, availablePermitsOnMsg);
+            }
+            if (availablePermitsOnByte == -1) {
+                availablePermitsOnByte = rgAvailablePermitsOnByte;
+            } else {
+                availablePermitsOnByte = Math.min(rgAvailablePermitsOnByte, availablePermitsOnByte);
             }
         }
 
-        return availablePermits;
+        availablePermitsOnMsg =
+                availablePermitsOnMsg != -1 ? Math.min(availablePermitsOnMsg, availablePermits) : availablePermits;
+        availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize);
+
+        availablePermitsOnByte =
+                availablePermitsOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, availablePermitsOnByte);
+
+        return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte);
     }
 
     protected void readMoreEntries() {
@@ -251,10 +310,10 @@ public abstract class PersistentReplicator extends AbstractReplicator
             log.info("[{}] Skip the reading due to new detected schema", replicatorId);
             return;
         }
-        int availablePermits = getAvailablePermits();
-
-        if (availablePermits > 0) {
-            int messagesToRead = Math.min(availablePermits, readBatchSize);
+        AvailablePermits availablePermits = getAvailablePermits();
+        if (availablePermits.isReadable()) {
+            int messagesToRead = availablePermits.getMessages();
+            long bytesToRead = availablePermits.getBytes();
             if (!isWritable()) {
                 if (log.isDebugEnabled()) {
                     log.debug("[{}] Throttling replication traffic because producer is not writable", replicatorId);
@@ -263,15 +322,13 @@ public abstract class PersistentReplicator extends AbstractReplicator
                 messagesToRead = 1;
             }
 
-            // If messagesToRead is 0 or less, correct it to 1 to prevent IllegalArgumentException
-            messagesToRead = Math.max(messagesToRead, 1);
-
             // Schedule read
             if (HAVE_PENDING_READ_UPDATER.compareAndSet(this, FALSE, TRUE)) {
                 if (log.isDebugEnabled()) {
-                    log.debug("[{}] Schedule read of {} messages", replicatorId, messagesToRead);
+                    log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, messagesToRead,
+                            bytesToRead);
                 }
-                cursor.asyncReadEntriesOrWait(messagesToRead, readMaxSizeBytes, this,
+                cursor.asyncReadEntriesOrWait(messagesToRead, bytesToRead, this,
                         null, topic.getMaxReadPosition());
             } else {
                 if (log.isDebugEnabled()) {
@@ -279,10 +336,9 @@ public abstract class PersistentReplicator extends AbstractReplicator
                             replicatorId, messagesToRead);
                 }
             }
-        } else if (availablePermits == -1) {
+        } else if (availablePermits.isExceeded()) {
             // no permits from rate limit
-            topic.getBrokerService().executor().schedule(
-                () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS);
+            scheduleReadMoreEntriesWhenExceeded();
         } else {
             if (log.isDebugEnabled()) {
                 log.debug("[{}] No Permits for reading. availablePermits: {}",
@@ -310,22 +366,56 @@ public abstract class PersistentReplicator extends AbstractReplicator
 
         readFailureBackoff.reduceToHalf();
 
-        boolean atLeastOneMessageSentForReplication = replicateEntries(entries);
+        ReplicationStatus replicationStatus = replicateEntries(entries);
 
         HAVE_PENDING_READ_UPDATER.set(this, FALSE);
 
-        if (atLeastOneMessageSentForReplication && !isWritable()) {
-            // Don't read any more entries until the current pending entries are persisted
-            if (log.isDebugEnabled()) {
-                log.debug("[{}] Pausing replication traffic. at-least-one: {} is-writable: {}", replicatorId,
-                        atLeastOneMessageSentForReplication, isWritable());
-            }
-        } else {
-            readMoreEntries();
+        switch (replicationStatus) {
+            case AT_LEAST_ONE_REPLICATED:
+                if (!isWritable()) {
+                    // Don't read any more entries until the current pending entries are persisted
+                    if (log.isDebugEnabled()) {
+                        log.debug("[{}] Pausing replication traffic. at-least-one: {} is-writable: {}", replicatorId,
+                                true, isWritable());
+                    }
+                } else {
+                    readMoreEntries();
+                }
+                break;
+            case RATE_LIMITED:
+                scheduleReadMoreEntriesWhenExceeded();
+                break;
+            case NO_ENTRIES_REPLICATED:
+                readMoreEntries();
+                break;
+            default:
+                throw new IllegalStateException("Unexpected replication status: " + replicationStatus);
         }
     }
 
-    protected abstract boolean replicateEntries(List<Entry> entries);
+    private void scheduleReadMoreEntriesWhenExceeded() {
+        topic.getBrokerService().executor().schedule(
+                () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS);
+    }
+
+    protected enum ReplicationStatus {
+        /**
+         * Replication was blocked due to rate limiting.
+         */
+        RATE_LIMITED,
+
+        /**
+         * At least one entry was successfully replicated.
+         */
+        AT_LEAST_ONE_REPLICATED,
+
+        /**
+         * No entries were replicated.
+         */
+        NO_ENTRIES_REPLICATED
+    }
+
+    protected abstract ReplicationStatus replicateEntries(List<Entry> entries);
 
     protected CompletableFuture<SchemaInfo> getSchemaInfo(MessageImpl msg) throws ExecutionException {
         if (msg.getSchemaVersion() == null || msg.getSchemaVersion().length == 0) {
@@ -576,7 +666,7 @@ public abstract class PersistentReplicator extends AbstractReplicator
                 exception.getMessage(), exception);
         if (exception instanceof CursorAlreadyClosedException) {
             log.warn("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already"
-                            + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception);
+                    + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception);
             // replicator is already deleted and cursor is already closed so, producer should also be disconnected.
             terminate();
             return;
@@ -605,6 +695,8 @@ public abstract class PersistentReplicator extends AbstractReplicator
         stats.replicationBacklog = cursor != null ? cursor.getNumberOfEntriesInBacklog(false) : 0;
         stats.connected = producer != null && producer.isConnected();
         stats.replicationDelayInSeconds = getReplicationDelayInSeconds();
+        stats.msgOutCounter = msgOutCounter.longValue();
+        stats.bytesOutCounter = bytesOutCounter.longValue();
 
         ProducerImpl producer = this.producer;
         if (producer != null) {
@@ -649,6 +741,11 @@ public abstract class PersistentReplicator extends AbstractReplicator
         return dispatchRateLimiter;
     }
 
+    @Override
+    public Optional<ResourceGroupDispatchLimiter> getResourceGroupDispatchRateLimiter() {
+        return resourceGroupDispatchRateLimiter;
+    }
+
     @Override
     public void initializeDispatchRateLimiterIfNeeded() {
         synchronized (dispatchRateLimiterLock) {
@@ -657,6 +754,20 @@ public abstract class PersistentReplicator extends AbstractReplicator
                 this.dispatchRateLimiter = Optional.of(new DispatchRateLimiter(topic, Type.REPLICATOR));
             }
         }
+        ResourceGroupService resourceGroupService = brokerService.getPulsar().getResourceGroupServiceManager();
+        HierarchyTopicPolicies hierarchyTopicPolicies = topic.getHierarchyTopicPolicies();
+        String resourceGroupName = hierarchyTopicPolicies.getResourceGroupName().get();
+        if (resourceGroupName != null) {
+            ResourceGroup resourceGroup = resourceGroupService.resourceGroupGet(resourceGroupName);
+            if (resourceGroup != null) {
+                resourceGroupDispatchRateLimiter = Optional.of(resourceGroup
+                        .getResourceGroupReplicationDispatchLimiter());
+            }
+        } else {
+            if (resourceGroupDispatchRateLimiter.isPresent()) {
+                resourceGroupDispatchRateLimiter = Optional.empty();
+            }
+        }
     }
 
     @Override
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 7faa86e5705..aa3bfa45087 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -426,7 +426,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                     if (!optPolicies.isPresent()) {
                         isEncryptionRequired = false;
                         updatePublishRateLimiter();
-                        updateResourceGroupLimiter(new Policies());
+                        updateResourceGroupLimiter();
                         initializeDispatchRateLimiterIfNeeded();
                         updateSubscribeRateLimiter();
                         return;
@@ -442,7 +442,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
 
                     updatePublishRateLimiter();
 
-                    updateResourceGroupLimiter(policies);
+                    updateResourceGroupLimiter();
 
                     this.isEncryptionRequired = policies.encryption_required;
 
@@ -1493,6 +1493,8 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
 
                                                     unregisterTopicPolicyListener();
 
+                                                    closeResourceGroupLimiter();
+
                                                     log.info("[{}] Topic deleted", topic);
                                                     deleteFuture.complete(null);
                                                 }
@@ -1703,6 +1705,9 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
                     subscribeRateLimiter.ifPresent(SubscribeRateLimiter::close);
 
                     unregisterTopicPolicyListener();
+
+                    closeResourceGroupLimiter();
+
                     log.info("[{}] Topic closed", topic);
                     cancelFencedTopicMonitoringTask();
                     closeFuture.complete(null);
@@ -3078,7 +3083,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
 
         // Apply policies for components.
         List<CompletableFuture<Void>> applyPolicyTasks = applyUpdatedTopicPolicies();
-        applyPolicyTasks.add(applyUpdatedNamespacePolicies(data));
+        applyPolicyTasks.add(applyUpdatedNamespacePolicies());
         return FutureUtil.waitForAll(applyPolicyTasks)
             .thenAccept(__ -> log.info("[{}] namespace-level policies updated successfully", topic))
             .exceptionally(ex -> {
@@ -3087,8 +3092,8 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
             });
     }
 
-    private CompletableFuture<Void> applyUpdatedNamespacePolicies(Policies namespaceLevelPolicies) {
-        return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter(namespaceLevelPolicies));
+    private CompletableFuture<Void> applyUpdatedNamespacePolicies() {
+        return FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter());
     }
 
     private List<CompletableFuture<Void>> applyUpdatedTopicPolicies() {
@@ -3107,6 +3112,7 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal
         applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateDispatchRateLimiter()));
         applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscribeRateLimiter()));
         applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updatePublishRateLimiter()));
+        applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateResourceGroupLimiter()));
 
         applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(() -> updateSubscriptionsDispatcherRateLimiter()));
         applyPoliciesFutureList.add(FutureUtil.runWithCurrentThread(
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
index b48f748bf5d..9fde570a042 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java
@@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.bookkeeper.mledger.Entry;
 import org.apache.bookkeeper.mledger.ManagedCursor;
 import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.client.impl.MessageIdImpl;
 import org.apache.pulsar.client.impl.MessageImpl;
@@ -54,8 +55,8 @@ public class ShadowReplicator extends PersistentReplicator {
     }
 
     @Override
-    protected boolean replicateEntries(List<Entry> entries) {
-        boolean atLeastOneMessageSentForReplication = false;
+    protected ReplicationStatus replicateEntries(List<Entry> entries) {
+        ReplicationStatus replicationStatus = ReplicationStatus.NO_ENTRIES_REPLICATED;
 
         try {
             // This flag is set to true when we skip at least one local message,
@@ -89,9 +90,26 @@ public class ShadowReplicator extends PersistentReplicator {
                     continue;
                 }
 
-                dispatchRateLimiter.ifPresent(rateLimiter -> rateLimiter.tryDispatchPermit(1, entry.getLength()));
+
+                int msgCount = msg.getMessageBuilder().hasNumMessagesInBatch()
+                        ? msg.getMessageBuilder().getNumMessagesInBatch() : 1;
+                dispatchRateLimiter.ifPresent(
+                        rateLimiter -> rateLimiter.tryDispatchPermit(msgCount, entry.getLength()));
+
+                ResourceGroupDispatchLimiter resourceGroupLimiter = resourceGroupDispatchRateLimiter.orElse(null);
+                if (resourceGroupLimiter != null) {
+                    if (!resourceGroupLimiter.tryAcquire(msgCount, entry.getLength())) {
+                        entry.release();
+                        msg.recycle();
+                        cursor.cancelPendingReadRequest();
+                        cursor.rewind();
+                        return ReplicationStatus.RATE_LIMITED;
+                    }
+                }
 
                 msgOut.recordEvent(headersAndPayload.readableBytes());
+                msgOutCounter.add(msgCount);
+                bytesOutCounter.add(length);
 
                 msg.setReplicatedFrom(localCluster);
 
@@ -102,13 +120,13 @@ public class ShadowReplicator extends PersistentReplicator {
                 // Increment pending messages for messages produced locally
                 PENDING_MESSAGES_UPDATER.incrementAndGet(this);
                 producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
-                atLeastOneMessageSentForReplication = true;
+                replicationStatus = ReplicationStatus.AT_LEAST_ONE_REPLICATED;
             }
         } catch (Exception e) {
             log.error("[{}] Unexpected exception in replication task for shadow topic: {}",
                     replicatorId, e.getMessage(), e);
         }
-        return atLeastOneMessageSentForReplication;
+        return replicationStatus;
     }
 
     /**
diff --git a/pulsar-broker/src/main/proto/ResourceUsage.proto b/pulsar-broker/src/main/proto/ResourceUsage.proto
index 4706c9dfbcd..ac7e1dea234 100644
--- a/pulsar-broker/src/main/proto/ResourceUsage.proto
+++ b/pulsar-broker/src/main/proto/ResourceUsage.proto
@@ -38,6 +38,7 @@ message ResourceUsage {
   optional NetworkUsage publish = 2;
   optional NetworkUsage dispatch = 3;
   optional StorageUsage storage = 4;
+  optional NetworkUsage replicationDispatch = 6;
 }
 
 message ResourceUsageInfo {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java
new file mode 100644
index 00000000000..4222862176b
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceGroupTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.broker.service.BrokerTestBase;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ResourceGroup;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Unit test {@link AdminResource}.
+ */
+@Test(groups = "broker-admin")
+public class AdminResourceGroupTest extends BrokerTestBase {
+
+    @BeforeClass
+    @Override
+    public void setup() throws Exception {
+        super.baseSetup();
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Override
+    protected void doInitConf() throws Exception {
+        super.doInitConf();
+        conf.setTopicLevelPoliciesEnabled(true);
+        conf.setSystemTopicEnabled(true);
+    }
+
+    @Test
+    public void testTopicResourceGroup() throws PulsarAdminException {
+        String topic = newTopicName();
+        TopicName topicName = TopicName.get(topic);
+
+        String resourceGroupName = "rg-topic-" + UUID.randomUUID();
+        ResourceGroup resourceGroup = new ResourceGroup();
+        resourceGroup.setPublishRateInMsgs(1000);
+        resourceGroup.setPublishRateInBytes(100000L);
+        resourceGroup.setDispatchRateInMsgs(2000);
+        resourceGroup.setDispatchRateInBytes(200000L);
+        admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+
+        admin.topics().createNonPartitionedTopic(topic);
+
+        admin.topicPolicies().setResourceGroup(topic, resourceGroupName);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(admin.topicPolicies().getResourceGroup(topic, true), resourceGroupName);
+        });
+
+        Awaitility.await().untilAsserted(() -> {
+            org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+                    .getTopicResourceGroup(topicName);
+            assertNotNull(rg);
+            assertEquals(rg.resourceGroupName, resourceGroupName);
+        });
+
+        admin.topicPolicies().removeResourceGroup(topic);
+        Awaitility.await().untilAsserted(() -> {
+            assertTrue(StringUtils.isEmpty(admin.topicPolicies().getResourceGroup(topic, true)));
+            org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+                    .getTopicResourceGroup(topicName);
+            assertNull(rg);
+        });
+    }
+
+    @Test
+    public void testTopicResourceGroupOverriderNamespaceResourceGroup() throws PulsarAdminException {
+        String namespaceResourceGroupName = "rg-ns-" + UUID.randomUUID();
+        ResourceGroup namespaceResourceGroup = new ResourceGroup();
+        namespaceResourceGroup.setPublishRateInMsgs(1001);
+        namespaceResourceGroup.setPublishRateInBytes(100001L);
+        namespaceResourceGroup.setDispatchRateInMsgs(2001);
+        namespaceResourceGroup.setDispatchRateInBytes(200001L);
+        admin.resourcegroups().createResourceGroup(namespaceResourceGroupName, namespaceResourceGroup);
+
+        String topicResourceGroupName = "rg-topic-" + UUID.randomUUID();
+        ResourceGroup topicResourceGroup = new ResourceGroup();
+        topicResourceGroup.setPublishRateInMsgs(1000);
+        topicResourceGroup.setPublishRateInBytes(100000L);
+        topicResourceGroup.setDispatchRateInMsgs(2000);
+        topicResourceGroup.setDispatchRateInBytes(200000L);
+        admin.resourcegroups().createResourceGroup(topicResourceGroupName, topicResourceGroup);
+
+        String topic = newTopicName();
+        TopicName topicName = TopicName.get(topic);
+        String namespace = topicName.getNamespace();
+        admin.namespaces().setNamespaceResourceGroup(namespace, namespaceResourceGroupName);
+        assertEquals(admin.namespaces().getNamespaceResourceGroup(namespace), namespaceResourceGroupName);
+
+        admin.topics().createNonPartitionedTopic(topic);
+        admin.topicPolicies().setResourceGroup(topic, topicResourceGroupName);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(admin.topicPolicies()
+                    .getResourceGroup(topic, true), topicResourceGroupName);
+            org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+                    .getTopicResourceGroup(topicName);
+            assertNotNull(rg);
+            assertEquals(rg.resourceGroupName, topicResourceGroupName);
+        });
+
+        admin.topicPolicies().removeResourceGroup(topic);
+        Awaitility.await().untilAsserted(() -> {
+            assertEquals(admin.topicPolicies()
+                    .getResourceGroup(topic, true), namespaceResourceGroupName);
+            org.apache.pulsar.broker.resourcegroup.ResourceGroup rg = pulsar.getResourceGroupServiceManager()
+                    .getTopicResourceGroup(topicName);
+            assertNull(rg);
+        });
+    }
+
+    @Test
+    public void testUpdateResourceGroup() throws PulsarAdminException {
+        String resourceGroupName = "rg-" + UUID.randomUUID();
+        ResourceGroup resourceGroup = new ResourceGroup();
+        resourceGroup.setPublishRateInMsgs(1000);
+        resourceGroup.setPublishRateInBytes(100000L);
+        resourceGroup.setDispatchRateInMsgs(2000);
+        resourceGroup.setDispatchRateInBytes(200000L);
+        resourceGroup.setReplicationDispatchRateInMsgs(10L);
+        resourceGroup.setReplicationDispatchRateInBytes(20L);
+
+        admin.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+        ResourceGroup got = admin.resourcegroups().getResourceGroup(resourceGroupName);
+        assertEquals(got, resourceGroup);
+
+        resourceGroup.setReplicationDispatchRateInMsgs(11L);
+        resourceGroup.setReplicationDispatchRateInBytes(29L);
+        admin.resourcegroups().updateResourceGroup(resourceGroupName, resourceGroup);
+        got = admin.resourcegroups().getResourceGroup(resourceGroupName);
+        assertEquals(got, resourceGroup);
+
+        admin.resourcegroups().deleteResourceGroup(resourceGroupName);
+        assertThrows(PulsarAdminException.NotFoundException.class,
+                () -> admin.resourcegroups().getResourceGroup(resourceGroupName));
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java
new file mode 100644
index 00000000000..e0142a74b3c
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupMetricTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertEquals;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
+import org.testng.annotations.Test;
+
+public class ResourceGroupMetricTest {
+    @Test
+    public void testLocalQuotaMetric() {
+        String rgName = "my-resource-group";
+        ResourceGroupMonitoringClass publish = ResourceGroupMonitoringClass.Publish;
+        int reportPeriod = 2;
+        BytesAndMessagesCount b = new BytesAndMessagesCount();
+        b.messages = 10;
+        b.bytes = 20;
+        int incTimes = 2;
+        for (int i = 0; i < 2; i++) {
+            ResourceGroupService.incRgCalculatedQuota(rgName, publish, b, reportPeriod);
+        }
+        double rgLocalUsageByteCount = ResourceGroupService.getRgQuotaByteCount(rgName, publish.name());
+        double rgQuotaMessageCount = ResourceGroupService.getRgQuotaMessageCount(rgName, publish.name());
+        assertEquals(rgLocalUsageByteCount, incTimes * b.bytes * reportPeriod);
+        assertEquals(rgQuotaMessageCount, incTimes * b.messages * reportPeriod);
+
+        double rgLocalUsageByte = ResourceGroupService.getRgQuotaByte(rgName, publish.name());
+        double rgQuotaMessage = ResourceGroupService.getRgQuotaMessage(rgName, publish.name());
+        assertEquals(rgLocalUsageByte, b.bytes);
+        assertEquals(rgQuotaMessage, b.messages);
+    }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java
new file mode 100644
index 00000000000..fc9862456d4
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterManagerTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.testng.annotations.Test;
+
+public class ResourceGroupRateLimiterManagerTest {
+
+    @Test
+    public void testNewReplicationDispatchRateLimiterWithEmptyResourceGroup() {
+        org.apache.pulsar.common.policies.data.ResourceGroup emptyResourceGroup =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+
+        @Cleanup(value = "shutdown")
+        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+        @Cleanup
+        ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(emptyResourceGroup,executorService);
+        assertFalse(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L);
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L);
+    }
+
+    @Test
+    public void testReplicationDispatchRateLimiterOnMsgs() {
+        org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInMsgs(10L);
+        @Cleanup(value = "shutdown")
+        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+        @Cleanup
+        ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+        assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), -1L);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), -1L);
+    }
+
+    @Test
+    public void testReplicationDispatchRateLimiterOnBytes() {
+        org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInBytes(20L);
+        @Cleanup(value = "shutdown")
+        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+        @Cleanup
+        ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+        assertTrue(resourceGroupDispatchLimiter.isDispatchRateLimitingEnabled());
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), -1L);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), -1L);
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+    }
+
+    @Test
+    public void testUpdateReplicationDispatchRateLimiter() {
+        org.apache.pulsar.common.policies.data.ResourceGroup resourceGroup =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInMsgs(10L);
+        resourceGroup.setReplicationDispatchRateInBytes(100L);
+
+        @Cleanup(value = "shutdown")
+        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
+        @Cleanup
+        ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = ResourceGroupRateLimiterManager.newReplicationDispatchRateLimiter(resourceGroup,executorService);
+
+        BytesAndMessagesCount quota = new BytesAndMessagesCount();
+        quota.messages = 20;
+        quota.bytes = 200;
+        ResourceGroupRateLimiterManager.updateReplicationDispatchRateLimiter(resourceGroupDispatchLimiter, quota);
+
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnByte(), quota.bytes);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), quota.bytes);
+        assertEquals(resourceGroupDispatchLimiter.getAvailableDispatchRateLimitOnMsg(), quota.messages);
+        assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), quota.messages);
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
index f47f7686743..0a9a071252f 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupServiceTest.java
@@ -18,25 +18,31 @@
  */
 package org.apache.pulsar.broker.resourcegroup;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Policy.Expiration;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
-import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields;
 import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerBrokerUsageStats;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.PerMonitoringClassFields;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
 import org.apache.pulsar.broker.service.resource.usage.NetworkUsage;
 import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
 import org.apache.pulsar.client.admin.PulsarAdminException;
 import org.apache.pulsar.common.naming.NamespaceName;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.ClusterData;
+import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
-import org.testng.Assert;
 import org.testng.annotations.Test;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
 public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
     @BeforeClass
@@ -51,7 +57,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
                                                   long currentMessagesUsed, long lastReportedMessages,
                                                   long lastReportTimeMSecsSinceEpoch)
             {
-                final int maxSuppressRounds = ResourceGroupService.MaxUsageReportSuppressRounds;
+                final int maxSuppressRounds = conf.getResourceUsageMaxUsageReportSuppressRounds();
                 if (++numLocalReportsEvaluated % maxSuppressRounds == (maxSuppressRounds - 1)) {
                     return true;
                 }
@@ -87,7 +93,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
     @Test
     public void measureOpsTime() throws PulsarAdminException {
         long mSecsStart, mSecsEnd, diffMsecs;
-        final int numPerfTestIterations = 10_000_000;
+        final int numPerfTestIterations = 1_000_000;
         org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
           new org.apache.pulsar.common.policies.data.ResourceGroup();
         BytesAndMessagesCount stats = new BytesAndMessagesCount();
@@ -111,16 +117,18 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
                 numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs)/numPerfTestIterations);
 
         // Going through the resource-group service
-        final String tenantName = "SomeTenant";
-        final String namespaceName = "SomeNameSpace";
+        final TopicName topicName = TopicName.get("SomeTenant/SomeNameSpace/my-topic");
+        rgs.registerTopic(rgName, topicName);
+        final String tenantName = topicName.getTenant();
+        final String namespaceName = topicName.getNamespace();
         rgs.registerTenant(rgName, tenantName);
-        final NamespaceName tenantAndNamespaceName = NamespaceName.get(tenantName, namespaceName);
+        final NamespaceName tenantAndNamespaceName = topicName.getNamespaceObject();
         rgs.registerNameSpace(rgName, tenantAndNamespaceName);
         mSecsStart = System.currentTimeMillis();
         for (int ix = 0; ix < numPerfTestIterations; ix++) {
             for (int monClassIdx = 0; monClassIdx < ResourceGroupMonitoringClass.values().length; monClassIdx++) {
                 monClass = ResourceGroupMonitoringClass.values()[monClassIdx];
-                rgs.incrementUsage(tenantName, namespaceName, monClass, stats);
+                rgs.incrementUsage(tenantName, namespaceName, topicName.toString(), monClass, stats);
             }
         }
         mSecsEnd = System.currentTimeMillis();
@@ -129,6 +137,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
                 numPerfTestIterations, diffMsecs, (1000 * (float) diffMsecs)/numPerfTestIterations);
         rgs.unRegisterTenant(rgName, tenantName);
         rgs.unRegisterNameSpace(rgName, tenantAndNamespaceName);
+        rgs.unRegisterTopic(topicName);
 
         // The overhead of a RG lookup
         mSecsStart = System.currentTimeMillis();
@@ -153,6 +162,8 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
         rgConfig.setPublishRateInMsgs(100);
         rgConfig.setDispatchRateInBytes(40000L);
         rgConfig.setDispatchRateInMsgs(500);
+        rgConfig.setReplicationDispatchRateInBytes(2000L);
+        rgConfig.setReplicationDispatchRateInMsgs(400L);
 
         int initialNumQuotaCalculations = numAnonymousQuotaCalculations;
         rgs.resourceGroupCreate(rgName, rgConfig);
@@ -167,6 +178,8 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
         rgConfig.setPublishRateInMsgs(rgConfig.getPublishRateInMsgs()*10);
         rgConfig.setDispatchRateInBytes(rgConfig.getDispatchRateInBytes()/10);
         rgConfig.setDispatchRateInMsgs(rgConfig.getDispatchRateInMsgs()/10);
+        rgConfig.setReplicationDispatchRateInBytes(rgConfig.getReplicationDispatchRateInBytes()/10);
+        rgConfig.setReplicationDispatchRateInMsgs(rgConfig.getReplicationDispatchRateInMsgs()/10);
         rgs.resourceGroupUpdate(rgName, rgConfig);
 
         Assert.assertEquals(rgs.getNumResourceGroups(), 1);
@@ -184,6 +197,9 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
         monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.Dispatch.ordinal()];
         Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getDispatchRateInBytes().longValue());
         Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getDispatchRateInMsgs().intValue());
+        monClassFields = retRG.monitoringClassFields[ResourceGroupMonitoringClass.ReplicationDispatch.ordinal()];
+        Assert.assertEquals(monClassFields.configValuesPerPeriod.bytes, rgConfig.getReplicationDispatchRateInBytes().longValue());
+        Assert.assertEquals(monClassFields.configValuesPerPeriod.messages, rgConfig.getReplicationDispatchRateInMsgs().intValue());
 
         Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(randomRgName));
 
@@ -197,6 +213,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
 
         final NamespaceName tenantAndNamespace = NamespaceName.get(tenantName, namespaceName);
         rgs.registerNameSpace(rgName, tenantAndNamespace);
+        rgs.registerTopic(rgName, topic);
 
         // Delete of our valid config should throw until we unref correspondingly.
         Assert.assertThrows(PulsarAdminException.class, () -> rgs.resourceGroupDelete(rgName));
@@ -213,6 +230,10 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
             // Gross hack!
             if (monClass == ResourceGroupMonitoringClass.Publish) {
                 nwUsage = usage.setPublish();
+            } else if (monClass == ResourceGroupMonitoringClass.Dispatch) {
+                nwUsage = usage.setDispatch();
+            } else if (monClass == ResourceGroupMonitoringClass.ReplicationDispatch) {
+                nwUsage = usage.setReplicationDispatch();
             } else {
                 nwUsage = usage.setDispatch();
             }
@@ -220,7 +241,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
             // maxUsageReportSuppressRounds iterations. So, if we run for maxUsageReportSuppressRounds iterations,
             // we should see needToReportLocalUsage() return true at least once.
             Set<Boolean> myBoolSet = new HashSet<>();
-            for (int idx = 0; idx < ResourceGroupService.MaxUsageReportSuppressRounds; idx++) {
+            for (int idx = 0; idx < conf.getResourceUsageMaxUsageReportSuppressRounds(); idx++) {
                 needToReport = retRG.setUsageInMonitoredEntity(monClass, nwUsage);
                 myBoolSet.add(needToReport);
             }
@@ -231,6 +252,7 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
 
         rgs.unRegisterTenant(rgName, tenantName);
         rgs.unRegisterNameSpace(rgName, tenantAndNamespace);
+        rgs.unRegisterTopic(topic);
 
         BytesAndMessagesCount publishQuota = rgs.getPublishRateLimiters(rgName);
 
@@ -255,6 +277,101 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
         Assert.assertThrows(PulsarAdminException.class, () -> rgs.getPublishRateLimiters(rgName));
 
         Assert.assertEquals(rgs.getNumResourceGroups(), 0);
+
+        Assert.assertEquals(rgs.getTopicConsumeStats().estimatedSize(), 0);
+        Assert.assertEquals(rgs.getTopicProduceStats().estimatedSize(), 0);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().estimatedSize(), 0);
+        Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+    }
+
+    @Test
+    public void testCleanupStatsWhenUnRegisterTopic()
+            throws PulsarAdminException {
+        String tenantName = UUID.randomUUID().toString();
+        org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        final String rgName = UUID.randomUUID().toString();
+        rgConfig.setPublishRateInBytes(15000L);
+        rgConfig.setPublishRateInMsgs(100);
+        rgConfig.setDispatchRateInBytes(40000L);
+        rgConfig.setDispatchRateInMsgs(500);
+        rgConfig.setReplicationDispatchRateInBytes(2000L);
+        rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+        rgs.resourceGroupCreate(rgName, rgConfig);
+        String nsName = tenantName + "/" + UUID.randomUUID();
+        TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID());
+        String topic = topicName.toString();
+
+        rgs.registerTopic(rgName, topicName);
+
+        // Simulate replicator
+        rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.ReplicationDispatch);
+        rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.Publish);
+        rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.Dispatch);
+        Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1);
+        Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1);
+        Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 1);
+        Set<String> replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement());
+        Assert.assertEquals(replicators.size(), 1);
+
+        rgs.unRegisterTopic(TopicName.get(topic));
+
+        Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+
+        rgs.resourceGroupDelete(rgName);
+    }
+
+    @Test
+    public void testCleanupStatsWhenUnRegisterNamespace()
+            throws PulsarAdminException {
+        String tenantName = UUID.randomUUID().toString();
+        org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        final String rgName = UUID.randomUUID().toString();
+        rgConfig.setPublishRateInBytes(15000L);
+        rgConfig.setPublishRateInMsgs(100);
+        rgConfig.setDispatchRateInBytes(40000L);
+        rgConfig.setDispatchRateInMsgs(500);
+        rgConfig.setReplicationDispatchRateInBytes(2000L);
+        rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+        rgs.resourceGroupCreate(rgName, rgConfig);
+        String nsName = tenantName + "/" + UUID.randomUUID();
+        TopicName topicName = TopicName.get(nsName + "/" + UUID.randomUUID());
+        String topic = topicName.toString();
+
+        rgs.registerNameSpace(rgName, topicName.getNamespaceObject());
+
+        // Simulate replicator
+        rgs.updateStatsWithDiff(topic, "remote-cluster", tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.ReplicationDispatch);
+        rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.Publish);
+        rgs.updateStatsWithDiff(topic, null, tenantName, nsName, 1, 1,
+                ResourceGroupMonitoringClass.Dispatch);
+        Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 1);
+        Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 1);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 1);
+        Set<String> replicators = rgs.getTopicToReplicatorsMap().get(rgs.getTopicToReplicatorsMap().keys().nextElement());
+        Assert.assertEquals(replicators.size(), 1);
+
+        rgs.unRegisterNameSpace(rgName, topicName.getNamespaceObject());
+
+        Assert.assertEquals(rgs.getTopicProduceStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getTopicConsumeStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getReplicationDispatchStats().asMap().size(), 0);
+        Assert.assertEquals(rgs.getTopicToReplicatorsMap().size(), 0);
+
+        rgs.resourceGroupDelete(rgName);
     }
 
     @Test
@@ -265,6 +382,75 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
         Assert.assertTrue(service.getCalculateQuotaPeriodicTask().isCancelled());
     }
 
+    private void assertTopicStatsCache(Cache<String, BytesAndMessagesCount> cache, long durationMS) {
+        Optional<Expiration<String, BytesAndMessagesCount>> expirationOptional =
+                cache.policy().expireAfterAccess();
+        Assert.assertTrue(expirationOptional.isPresent());
+        Expiration<String, BytesAndMessagesCount> expiration = expirationOptional.get();
+        Assert.assertEquals(expiration.getExpiresAfter().toMillis(), durationMS);
+    }
+
+    @Test
+    public void testTopicStatsCache() {
+        long ms = 2_000;
+        Cache<String, BytesAndMessagesCount> cache =
+                pulsar.getResourceGroupServiceManager().newStatsCache(TimeUnit.MILLISECONDS.toMillis(ms));
+        String key = "topic-1";
+        BytesAndMessagesCount value = new BytesAndMessagesCount();
+        cache.put(key, value);
+        Assert.assertEquals(cache.getIfPresent(key), value);
+        Awaitility.await().pollDelay(ms + 200 , TimeUnit.MILLISECONDS).untilAsserted(() -> {
+            Assert.assertNull(cache.getIfPresent(key));
+        });
+
+        long expMS =
+                TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs())
+                        * 2;
+        assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicConsumeStats(), expMS);
+        assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getTopicProduceStats(), expMS);
+        assertTopicStatsCache(pulsar.getResourceGroupServiceManager().getReplicationDispatchStats(), expMS);
+    }
+
+    @Test
+    public void testBrokerStatsCache() throws PulsarAdminException {
+        long ms = 2_000;
+        String key = "broker-1";
+        PerBrokerUsageStats value = new PerBrokerUsageStats();
+        PerMonitoringClassFields perMonitoringClassFields = PerMonitoringClassFields.create(ms);
+        Cache<String, PerBrokerUsageStats> usageFromOtherBrokers = perMonitoringClassFields.usageFromOtherBrokers;
+        usageFromOtherBrokers.put(key, value);
+        Assert.assertEquals(usageFromOtherBrokers.getIfPresent(key), value);
+        Awaitility.await().atMost(ms + 500, TimeUnit.MILLISECONDS).untilAsserted(() -> {
+            Assert.assertNull(usageFromOtherBrokers.getIfPresent(key));
+        });
+
+        org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        final String rgName = UUID.randomUUID().toString();
+        rgConfig.setPublishRateInBytes(15000L);
+        rgConfig.setPublishRateInMsgs(100);
+        rgConfig.setDispatchRateInBytes(40000L);
+        rgConfig.setDispatchRateInMsgs(500);
+        rgConfig.setReplicationDispatchRateInBytes(2000L);
+        rgConfig.setReplicationDispatchRateInMsgs(400L);
+
+        ResourceGroupService resourceGroupServiceManager = pulsar.getResourceGroupServiceManager();
+        resourceGroupServiceManager.resourceGroupCreate(rgName, rgConfig);
+        ResourceGroup resourceGroup = resourceGroupServiceManager.resourceGroupGet(rgName);
+        PerMonitoringClassFields publishMonitoredEntity =
+                resourceGroup.getMonitoredEntity(ResourceGroupMonitoringClass.Publish);
+        Cache<String, PerBrokerUsageStats> cache = publishMonitoredEntity.usageFromOtherBrokers;
+        Optional<Expiration<String, PerBrokerUsageStats>> expirationOptional =
+                cache.policy().expireAfterWrite();
+        Assert.assertTrue(expirationOptional.isPresent());
+        Expiration<String, PerBrokerUsageStats> brokerUsageStatsExpiration = expirationOptional.get();
+
+        long statsDuration =
+                TimeUnit.SECONDS.toMillis(pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs())
+                        * conf.getResourceUsageMaxUsageReportSuppressRounds() * 2;
+        Assert.assertEquals(brokerUsageStatsExpiration.getExpiresAfter().toMillis(), statsDuration);
+    }
+
     private ResourceGroupService rgs;
     int numAnonymousQuotaCalculations;
 
@@ -272,7 +458,6 @@ public class ResourceGroupServiceTest extends MockedPulsarServiceBaseTest {
 
     private static final int PUBLISH_INTERVAL_SECS = 500;
     private void prepareData() throws PulsarAdminException {
-        this.conf.setResourceUsageTransportPublishIntervalInSecs(PUBLISH_INTERVAL_SECS);
         admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build());
     }
 }
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java
new file mode 100644
index 00000000000..fa87b2ef705
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationOnTopicLevelTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.resourcegroup;
+
+import static org.testng.Assert.assertNotNull;
+import com.google.common.collect.Sets;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroup.ResourceGroupMonitoringClass;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupService.ResourceGroupUsageStatsType;
+import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.broker.service.resource.usage.ResourceUsage;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.SubscriptionInitialPosition;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+public class ResourceGroupUsageAggregationOnTopicLevelTest extends ProducerConsumerBase {
+
+    private final String TenantName = "pulsar-test";
+    private final String NsName = "test";
+    private final String TenantAndNsName = TenantName + "/" + NsName;
+    private final String TestProduceConsumeTopicName = "/test/prod-cons-topic";
+    private final String PRODUCE_CONSUME_PERSISTENT_TOPIC = "persistent://" + TenantAndNsName + TestProduceConsumeTopicName;
+    private final String PRODUCE_CONSUME_NON_PERSISTENT_TOPIC =
+            "non-persistent://" + TenantAndNsName + TestProduceConsumeTopicName;
+
+    @BeforeClass(alwaysRun = true)
+    @Override
+    protected void setup() throws Exception {
+        super.internalSetup();
+        this.conf.setAllowAutoTopicCreation(true);
+
+        final String clusterName = "test";
+        admin.clusters().createCluster(clusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build());
+        admin.tenants().createTenant(TenantName,
+                new TenantInfoImpl(Sets.newHashSet("fakeAdminRole"), Sets.newHashSet(clusterName)));
+        admin.namespaces().createNamespace(TenantAndNsName);
+        admin.namespaces().setNamespaceReplicationClusters(TenantAndNsName, Sets.newHashSet(clusterName));
+    }
+
+    @AfterClass(alwaysRun = true)
+    @Override
+    protected void cleanup() throws Exception {
+        super.internalCleanup();
+    }
+
+    @Test
+    public void testPersistentTopicProduceConsumeUsageOnRG() throws Exception {
+        testProduceConsumeUsageOnRG(PRODUCE_CONSUME_PERSISTENT_TOPIC);
+    }
+    
+    @Test
+    public void testNonPersistentTopicProduceConsumeUsageOnRG() throws Exception {
+        testProduceConsumeUsageOnRG(PRODUCE_CONSUME_NON_PERSISTENT_TOPIC);
+    }
+
+    private void testProduceConsumeUsageOnRG(String topicString) throws Exception {
+        ResourceQuotaCalculator dummyQuotaCalc = new ResourceQuotaCalculator() {
+            @Override
+            public boolean needToReportLocalUsage(long currentBytesUsed, long lastReportedBytes,
+                                                  long currentMessagesUsed, long lastReportedMessages,
+                                                  long lastReportTimeMSecsSinceEpoch) {
+                return false;
+            }
+
+            @Override
+            public long computeLocalQuota(long confUsage, long myUsage, long[] allUsages) {
+                return 0;
+            }
+        };
+
+        @Cleanup
+        ResourceUsageTopicTransportManager transportMgr = new ResourceUsageTopicTransportManager(pulsar);
+        @Cleanup
+        ResourceGroupService rgs = new ResourceGroupService(pulsar, TimeUnit.MILLISECONDS, transportMgr,
+                dummyQuotaCalc);
+
+        String activeRgName = "runProduceConsume";
+        ResourceGroup activeRG;
+
+        ResourceUsagePublisher ruP = new ResourceUsagePublisher() {
+            @Override
+            public String getID() {
+                return rgs.resourceGroupGet(activeRgName).resourceGroupName;
+            }
+
+            @Override
+            public void fillResourceUsage(ResourceUsage resourceUsage) {
+                rgs.resourceGroupGet(activeRgName).rgFillResourceUsage(resourceUsage);
+            }
+        };
+
+        ResourceUsageConsumer ruC = new ResourceUsageConsumer() {
+            @Override
+            public String getID() {
+                return rgs.resourceGroupGet(activeRgName).resourceGroupName;
+            }
+
+            @Override
+            public void acceptResourceUsage(String broker, ResourceUsage resourceUsage) {
+                rgs.resourceGroupGet(activeRgName).rgResourceUsageListener(broker, resourceUsage);
+            }
+        };
+
+        org.apache.pulsar.common.policies.data.ResourceGroup rgConfig =
+                new org.apache.pulsar.common.policies.data.ResourceGroup();
+        rgConfig.setPublishRateInBytes(1500L);
+        rgConfig.setPublishRateInMsgs(100);
+        rgConfig.setDispatchRateInBytes(4000L);
+        rgConfig.setPublishRateInMsgs(500);
+
+        rgs.resourceGroupCreate(activeRgName, rgConfig, ruP, ruC);
+
+        activeRG = rgs.resourceGroupGet(activeRgName);
+        assertNotNull(activeRG);
+
+        String subscriptionName = "my-subscription";
+        @Cleanup
+        Consumer<byte[]> consumer = pulsarClient.newConsumer()
+                .topic(topicString)
+                .subscriptionName(subscriptionName)
+                .subscriptionType(SubscriptionType.Shared)
+                .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+                .subscribe();
+
+        @Cleanup
+        Producer<byte[]> producer = pulsarClient.newProducer()
+                .topic(topicString)
+                .create();
+
+        TopicName myTopic = TopicName.get(topicString);
+        rgs.unRegisterTopic(myTopic);
+        rgs.registerTopic(activeRgName,myTopic);
+
+        final int NumMessagesToSend = 10;
+        int sentNumBytes = 0;
+        int sentNumMsgs = 0;
+        for (int ix = 0; ix < NumMessagesToSend; ix++) {
+            byte[] mesg = String.format("Hi, ix=%s", ix).getBytes();
+            producer.send(mesg);
+            sentNumBytes += mesg.length;
+            sentNumMsgs++;
+        }
+
+        this.verifyStats(rgs, topicString, activeRgName, sentNumBytes, sentNumMsgs, 0, 0,
+                true, false);
+
+        int recvdNumBytes = 0;
+        int recvdNumMsgs = 0;
+
+        Message<byte[]> message;
+        while (recvdNumMsgs < sentNumMsgs) {
+            message = consumer.receive();
+            recvdNumBytes += message.getValue().length;
+            recvdNumMsgs++;
+        }
+
+        this.verifyStats(rgs,topicString, activeRgName, sentNumBytes, sentNumMsgs, recvdNumBytes, recvdNumMsgs,
+                true, true);
+    }
+
+    private void verifyStats(ResourceGroupService rgs, String topicString, String rgName,
+                             int sentNumBytes, int sentNumMsgs,
+                             int recvdNumBytes, int recvdNumMsgs,
+                             boolean checkProduce, boolean checkConsume) throws PulsarAdminException {
+        BrokerService bs = pulsar.getBrokerService();
+        Awaitility.await().untilAsserted(() -> {
+            TopicStatsImpl topicStats = bs.getTopicStats().get(topicString);
+            assertNotNull(topicStats);
+            if (checkProduce) {
+                Assert.assertTrue(topicStats.bytesInCounter >= sentNumBytes);
+                Assert.assertEquals(sentNumMsgs, topicStats.msgInCounter);
+            }
+            if (checkConsume) {
+                Assert.assertTrue(topicStats.bytesOutCounter >= recvdNumBytes);
+                Assert.assertEquals(recvdNumMsgs, topicStats.msgOutCounter);
+            }
+        });
+        if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
+            rgs.aggregateResourceGroupLocalUsages();
+            BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish,
+                    ResourceGroupUsageStatsType.Cumulative);
+            BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch,
+                    ResourceGroupUsageStatsType.Cumulative);
+
+            if (checkProduce) {
+                Assert.assertTrue(prodCounts.bytes >= sentNumBytes);
+                Assert.assertEquals(sentNumMsgs, prodCounts.messages);
+            }
+            if (checkConsume) {
+                Assert.assertTrue(consCounts.bytes >= recvdNumBytes);
+                Assert.assertEquals(recvdNumMsgs, consCounts.messages);
+            }
+        }
+    }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
index 72fd91ea46d..84c4926b5f9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupUsageAggregationTest.java
@@ -191,8 +191,6 @@ public class ResourceGroupUsageAggregationTest extends ProducerConsumerBase {
                 }
             }
         }
-        rgs.getTopicConsumeStats().clear();
-        rgs.getTopicProduceStats().clear();
 
         rgs.unRegisterTenant(activeRgName, tenantString);
         rgs.unRegisterNameSpace(activeRgName, NamespaceName.get(nsString));
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
index eeda957bf3e..73033f36530 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceQuotaCalculatorImplTest.java
@@ -32,7 +32,7 @@ public class ResourceQuotaCalculatorImplTest extends MockedPulsarServiceBaseTest
     @Override
     protected void setup() throws Exception {
         super.internalSetup();
-        this.rqCalc = new ResourceQuotaCalculatorImpl();
+        this.rqCalc = new ResourceQuotaCalculatorImpl(pulsar);
     }
 
     @AfterClass(alwaysRun = true)
@@ -121,5 +121,16 @@ public class ResourceQuotaCalculatorImplTest extends MockedPulsarServiceBaseTest
         Assert.assertTrue(rqCalc.needToReportLocalUsage(940, 1000, 94, 100, System.currentTimeMillis()));
     }
 
+    @Test
+    public void testUsedUsageLessThanConfigUsage() throws PulsarAdminException {
+        final long config = 100;
+        final long usedUsage = 100 / pulsar.getConfiguration().getResourceUsageTransportPublishIntervalInSecs() / 3;
+        final long[] allUsage = {usedUsage, usedUsage};
+        for (long n : allUsage) {
+            long localQuota = rqCalc.computeLocalQuota(config, n, allUsage);
+            Assert.assertEquals(localQuota, config);
+        }
+    }
+
     private ResourceQuotaCalculatorImpl rqCalc;
 }
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
index 8cf5511a684..9eacb76f371 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceUsageTransportManagerTest.java
@@ -80,7 +80,7 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
                 resourceUsage.setOwner(getID());
                 resourceUsage.setPublish().setMessagesPerPeriod(1000).setBytesPerPeriod(10001);
                 resourceUsage.setStorage().setTotalBytes(500003);
-
+                resourceUsage.setReplicationDispatch().setMessagesPerPeriod(2000).setBytesPerPeriod(4000);
             }
         };
 
@@ -99,6 +99,10 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
                 p.setBytesPerPeriod(resourceUsage.getPublish().getBytesPerPeriod());
                 p.setMessagesPerPeriod(resourceUsage.getPublish().getMessagesPerPeriod());
 
+                p = recvdUsage.setReplicationDispatch();
+                p.setBytesPerPeriod(resourceUsage.getReplicationDispatch().getBytesPerPeriod());
+                p.setMessagesPerPeriod(resourceUsage.getReplicationDispatch().getMessagesPerPeriod());
+
                 recvdUsage.setStorage().setTotalBytes(resourceUsage.getStorage().getTotalBytes());
             }
         };
@@ -113,6 +117,8 @@ public class ResourceUsageTransportManagerTest extends MockedPulsarServiceBaseTe
         assertNotNull(recvdUsage.getStorage());
         assertEquals(recvdUsage.getPublish().getBytesPerPeriod(), 10001);
         assertEquals(recvdUsage.getStorage().getTotalBytes(), 500003);
+        assertEquals(recvdUsage.getReplicationDispatch().getBytesPerPeriod(), 4000);
+        assertEquals(recvdUsage.getReplicationDispatch().getMessagesPerPeriod(), 2000);
     }
 
     private void prepareData() throws PulsarServerException, PulsarAdminException, PulsarClientException {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
index b97210b009a..4289b5ecca5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorRateLimiterTest.java
@@ -18,21 +18,26 @@
  */
 package org.apache.pulsar.broker.service;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 import static org.testng.AssertJUnit.assertFalse;
 import com.google.common.collect.Sets;
 import java.lang.reflect.Method;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import lombok.Cleanup;
+import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter;
 import org.apache.pulsar.broker.service.persistent.PersistentTopic;
 import org.apache.pulsar.client.api.Consumer;
 import org.apache.pulsar.client.api.MessageRoutingMode;
 import org.apache.pulsar.client.api.Producer;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.common.policies.data.DispatchRate;
+import org.apache.pulsar.common.policies.data.ResourceGroup;
 import org.awaitility.Awaitility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -98,6 +103,7 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
 
         // rate limiter disable by default
         assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent());
+        assertFalse(topic.getReplicators().values().get(0).getResourceGroupDispatchRateLimiter().isPresent());
 
         //set topic-level policy, which should take effect
         DispatchRate topicRate = DispatchRate.builder()
@@ -119,6 +125,24 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
         assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1);
         assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(),
             -1L);
+
+        // ResourceGroupDispatchRateLimiter
+        String resourceGroupName = UUID.randomUUID().toString();
+        ResourceGroup resourceGroup = new ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInBytes(10L);
+        resourceGroup.setReplicationDispatchRateInMsgs(20L);
+        admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+        Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+                .getResourceGroup(resourceGroupName)));
+        admin1.topicPolicies().setResourceGroup(topicName, resourceGroupName);
+
+        Replicator replicator = topic.getReplicators().values().get(0);
+        Awaitility.await().untilAsserted(() -> {
+            assertTrue(replicator.getResourceGroupDispatchRateLimiter().isPresent());
+            ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = replicator.getResourceGroupDispatchRateLimiter().get();
+            assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+            assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+        });
     }
 
     @Test
@@ -142,6 +166,7 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
 
         // rate limiter disable by default
         assertFalse(topic.getReplicators().values().get(0).getRateLimiter().isPresent());
+        assertFalse(topic.getReplicators().values().get(0).getResourceGroupDispatchRateLimiter().isPresent());
 
         //set namespace-level policy, which should take effect
         DispatchRate topicRate = DispatchRate.builder()
@@ -163,6 +188,24 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
         assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), -1);
         assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(),
             -1L);
+
+        // ResourceGroupDispatchRateLimiter
+        String resourceGroupName = UUID.randomUUID().toString();
+        ResourceGroup resourceGroup = new ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInBytes(10L);
+        resourceGroup.setReplicationDispatchRateInMsgs(20L);
+        admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+        Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+                .getResourceGroup(resourceGroupName)));
+        admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName);
+
+        Replicator replicator = topic.getReplicators().values().get(0);
+        Awaitility.await().untilAsserted(() -> {
+            assertTrue(replicator.getResourceGroupDispatchRateLimiter().isPresent());
+            ResourceGroupDispatchLimiter resourceGroupDispatchLimiter = replicator.getResourceGroupDispatchRateLimiter().get();
+            assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnByte(), resourceGroup.getReplicationDispatchRateInBytes().longValue());
+            assertEquals(resourceGroupDispatchLimiter.getDispatchRateOnMsg(), resourceGroup.getReplicationDispatchRateInMsgs().longValue());
+        });
     }
 
     @Test
@@ -545,5 +588,114 @@ public class ReplicatorRateLimiterTest extends ReplicatorTestBase {
         producer.close();
     }
 
+    @Test
+    public void testResourceGroupReplicatorRateLimiter() throws Exception {
+        final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis();
+        final String topicName = "persistent://" + namespace + "/" + UUID.randomUUID();
+
+        admin1.namespaces().createNamespace(namespace);
+        // 0. set 2 clusters, there will be 1 replicator in each topic
+        admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
+
+        // ResourceGroupDispatchRateLimiter
+        int messageRate = 100;
+        String resourceGroupName = UUID.randomUUID().toString();
+        ResourceGroup resourceGroup = new ResourceGroup();
+        resourceGroup.setReplicationDispatchRateInMsgs((long) messageRate);
+        admin1.resourcegroups().createResourceGroup(resourceGroupName, resourceGroup);
+        Awaitility.await().untilAsserted(() -> assertNotNull(admin1.resourcegroups()
+                .getResourceGroup(resourceGroupName)));
+        admin1.namespaces().setNamespaceResourceGroup(namespace, resourceGroupName);
+
+        @Cleanup
+        PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+
+        @Cleanup
+        Producer<byte[]> producer = client1.newProducer().topic(topicName)
+                .enableBatching(false)
+                .messageRoutingMode(MessageRoutingMode.SinglePartition)
+                .create();
+
+        @Cleanup
+        PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS)
+                .build();
+        final AtomicInteger totalReceived = new AtomicInteger(0);
+
+        @Cleanup
+        Consumer<byte[]> consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> {
+            Assert.assertNotNull(msg, "Message cannot be null");
+            String receivedMessage = new String(msg.getData());
+            log.debug("Received message [{}] in the listener", receivedMessage);
+            totalReceived.incrementAndGet();
+        }).subscribe();
+
+        int numMessages = 500;
+        for (int i = 0; i < numMessages; i++) {
+            producer.send(new byte[80]);
+        }
+
+        Assert.assertTrue(totalReceived.get() < messageRate * 2);
+    }
+
+    @Test
+    public void testReplicatorRateLimiterByBytes() throws Exception {
+        final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis();
+        final String topicName = "persistent://" + namespace + "/RateLimiterByBytes";
+
+        admin1.namespaces().createNamespace(namespace);
+        // 0. set 2 clusters, there will be 1 replicator in each topic
+        admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
+
+        final int byteRate = 400;
+        final int payloadSize = 100;
+        DispatchRate dispatchRate = DispatchRate.builder()
+                .dispatchThrottlingRateInMsg(-1)
+                .dispatchThrottlingRateInByte(byteRate)
+                .ratePeriodInSecond(360)
+                .build();
+        admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate);
+
+        @Cleanup
+        PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
+        @Cleanup
+        Producer<byte[]> producer = client1.newProducer().topic(topicName)
+                .enableBatching(false)
+                .messageRoutingMode(MessageRoutingMode.SinglePartition)
+                .create();
+
+        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get();
+
+        Awaitility.await()
+                .untilAsserted(() -> assertTrue(topic.getReplicators().values().get(0).getRateLimiter().isPresent()));
+        assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), byteRate);
+
+        @Cleanup
+        PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString())
+                .build();
+        final AtomicInteger totalReceived = new AtomicInteger(0);
+
+        @Cleanup
+        Consumer<byte[]> ignored = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2")
+                .messageListener((c1, msg) -> {
+                    Assert.assertNotNull(msg, "Message cannot be null");
+                    String receivedMessage = new String(msg.getData());
+                    log.debug("Received message [{}] in the listener", receivedMessage);
+                    totalReceived.incrementAndGet();
+                }).subscribe();
+
+        // The total bytes is 5 times the rate limit value.
+        int numMessages = byteRate / payloadSize * 5;
+        for (int i = 0; i < numMessages * payloadSize; i++) {
+            producer.send(new byte[payloadSize]);
+        }
+
+        Awaitility.await().pollDelay(5, TimeUnit.SECONDS)
+                .untilAsserted(() -> {
+                    // The rate limit occurs in the next reading cycle, so a value fault tolerance needs to be added.
+                    assertThat(totalReceived.get()).isLessThan((byteRate / payloadSize) + 2);
+                });
+    }
+
     private static final Logger log = LoggerFactory.getLogger(ReplicatorRateLimiterTest.class);
 }
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
index 843976eb13d..52eb9cfe317 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/ResourceGroups.java
@@ -75,6 +75,8 @@ public interface ResourceGroups {
          *     "PublishRateInBytes" : "value",
          *     "DispatchRateInMsgs" : "value",
          *     "DispatchRateInBytes" : "value"
+         *     "ReplicationDispatchRateInMsgs" : "value"
+         *     "ReplicationDispatchRateInBytes" : "value"
          * </code>
          * </pre>
          *
@@ -101,6 +103,8 @@ public interface ResourceGroups {
          *     "PublishRateInBytes" : "value",
          *     "DispatchRateInMsgs" : "value",
          *     "DspatchRateInBytes" : "value"
+         *     "ReplicationDispatchRateInMsgs" : "value"
+         *     "ReplicationDispatchRateInBytes" : "value"
          * </code>
          * </pre>
          *
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
index f6cd2a5a0ef..a771478f146 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/TopicPolicies.java
@@ -21,6 +21,8 @@ package org.apache.pulsar.client.admin;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
 import org.apache.pulsar.client.api.SubscriptionType;
 import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride;
 import org.apache.pulsar.common.policies.data.BacklogQuota;
@@ -1913,4 +1915,61 @@ public interface TopicPolicies {
      *            Topic name
      */
     CompletableFuture<Void> removeAutoSubscriptionCreationAsync(String topic);
+
+    /**
+     * Get the ResourceGroup for a topic.
+     *
+     * @param topic Topic name
+     * @param applied True gets namespace level configuration if ResourceGroup does not exist on the topic.
+     *                False gets topic level configuration.
+     * @return ResourceGroup
+     * @throws NotAuthorizedException Don't have admin permission
+     * @throws NotFoundException Topic does not exist
+     * @throws PulsarAdminException Unexpected error
+     */
+    String getResourceGroup(String topic, boolean applied) throws PulsarAdminException;
+
+    /**
+     * Get the ResourceGroup for a topic asynchronously.
+     *
+     * @param topic Topic name
+     */
+    CompletableFuture<String> getResourceGroupAsync(String topic, boolean applied);
+
+    /**
+     * Set the ResourceGroup for a topic.
+     *
+     * @param topic Topic name
+     * @param resourceGroupName ResourceGroup name
+     * @throws NotAuthorizedException Don't have admin permission
+     * @throws NotFoundException Topic does not exist
+     * @throws PulsarAdminException Unexpected error
+     */
+    void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException;
+
+    /**
+     * Set the ResourceGroup for a topic.
+     *
+     * @param topic Topic name
+     * @param resourceGroupName ResourceGroup name
+     */
+    CompletableFuture<Void> setResourceGroupAsync(String topic, String resourceGroupName);
+
+    /**
+     * Remove the ResourceGroup on a topic.
+     *
+     * @param topic Topic name
+     * @throws NotAuthorizedException Don't have admin permission
+     * @throws NotFoundException Topic does not exist
+     * @throws PulsarAdminException Unexpected error
+     */
+    void removeResourceGroup(String topic) throws PulsarAdminException;
+
+    /**
+     * Remove the ResourceGroup on a topic asynchronously.
+     *
+     * @param topic Topic name
+     */
+    CompletableFuture<Void> removeResourceGroupAsync(String topic);
 }
+
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
index 24be2f9380b..870725021d4 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ReplicatorStats.java
@@ -35,6 +35,16 @@ public interface ReplicatorStats {
     /** Total throughput delivered to the replication-subscriber (bytes/s). */
     double getMsgThroughputOut();
 
+    /**
+     * Total bytes delivered to the replication-subscriber (bytes).
+     */
+    long getBytesOutCounter();
+
+    /**
+     * Total messages delivered to the replication-subscriber (msg).
+     */
+    long getMsgOutCounter();
+
     /** Total rate of messages expired (msg/s). */
     double getMsgRateExpired();
 
diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
index bd47427e9aa..81e06107a8f 100644
--- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
+++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ResourceGroup.java
@@ -26,4 +26,7 @@ public class ResourceGroup {
     private Long publishRateInBytes;
     private Integer dispatchRateInMsgs;
     private Long dispatchRateInBytes;
+
+    private Long replicationDispatchRateInMsgs;
+    private Long replicationDispatchRateInBytes;
 }
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
index 915b22a2589..1af249a969b 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicPoliciesImpl.java
@@ -22,6 +22,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.InvocationCallback;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.GenericType;
 import javax.ws.rs.core.MediaType;
@@ -1260,6 +1261,54 @@ public class TopicPoliciesImpl extends BaseResource implements TopicPolicies {
         return asyncDeleteRequest(path);
     }
 
+    @Override
+    public String getResourceGroup(String topic, boolean applied) throws PulsarAdminException {
+        return sync(() -> getResourceGroupAsync(topic, applied));
+    }
+
+    @Override
+    public CompletableFuture<String> getResourceGroupAsync(String topic, boolean applied) {
+        TopicName tn = validateTopic(topic);
+        WebTarget path = topicPath(tn, "resourceGroup");
+        path = path.queryParam("applied", applied);
+        final CompletableFuture<String> future = new CompletableFuture<>();
+        asyncGetRequest(path,
+                new InvocationCallback<String>() {
+                    @Override
+                    public void completed(String rgName) {
+                        future.complete(rgName);
+                    }
+
+                    @Override
+                    public void failed(Throwable throwable) {
+                        future.completeExceptionally(getApiException(throwable.getCause()));
+                    }
+                });
+        return future;
+    }
+
+    @Override
+    public void setResourceGroup(String topic, String resourceGroupName) throws PulsarAdminException {
+        sync(() -> setResourceGroupAsync(topic, resourceGroupName));
+    }
+
+    @Override
+    public CompletableFuture<Void> setResourceGroupAsync(String topic, String resourceGroupName) {
+        TopicName tn = validateTopic(topic);
+        WebTarget path = topicPath(tn, "resourceGroup");
+        return asyncPostRequest(path, Entity.entity(resourceGroupName, MediaType.APPLICATION_JSON_TYPE));
+    }
+
+    @Override
+    public void removeResourceGroup(String topic) throws PulsarAdminException {
+        sync(() -> removeResourceGroupAsync(topic));
+    }
+
+    @Override
+    public CompletableFuture<Void> removeResourceGroupAsync(String topic) {
+        return setResourceGroupAsync(topic, null);
+    }
+
     /*
      * returns topic name with encoded Local Name
      */
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
index 7ed853be44d..6e62f57218f 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdResourceGroups.java
@@ -72,6 +72,14 @@ public class CmdResourceGroups extends CmdBase {
                 + "(default -1 will be overwrite if not passed)", required = false)
         private Long dispatchRateInBytes;
 
+        @Parameter(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate "
+                + "(default -1 will be overwrite if not passed)", required = false)
+        private Long replicationDispatchRateInMsgs;
+
+        @Parameter(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate "
+                + "(default -1 will be overwrite if not passed)", required = false)
+        private Long replicationDispatchRateInBytes;
+
         @Override
         void run() throws PulsarAdminException {
             String name = getOneArgument(params);
@@ -81,6 +89,8 @@ public class CmdResourceGroups extends CmdBase {
             resourcegroup.setDispatchRateInBytes(dispatchRateInBytes);
             resourcegroup.setPublishRateInMsgs(publishRateInMsgs);
             resourcegroup.setPublishRateInBytes(publishRateInBytes);
+            resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs);
+            resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes);
             getAdmin().resourcegroups().createResourceGroup(name, resourcegroup);
         }
     }
@@ -107,6 +117,14 @@ public class CmdResourceGroups extends CmdBase {
                 "-bd" }, description = "byte-dispatch-rate ", required = false)
         private Long dispatchRateInBytes;
 
+        @Parameter(names = {"--replication-msg-dispatch-rate"}, description = "replication-msg-dispatch-rate "
+                + "(default -1 will be overwrite if not passed)", required = false)
+        private Long replicationDispatchRateInMsgs;
+
+        @Parameter(names = {"--replication-byte-dispatch-rate"}, description = "replication-byte-dispatch-rate "
+                + "(default -1 will be overwrite if not passed)", required = false)
+        private Long replicationDispatchRateInBytes;
+
         @Override
         void run() throws PulsarAdminException {
             String name = getOneArgument(params);
@@ -116,6 +134,8 @@ public class CmdResourceGroups extends CmdBase {
             resourcegroup.setDispatchRateInBytes(dispatchRateInBytes);
             resourcegroup.setPublishRateInMsgs(publishRateInMsgs);
             resourcegroup.setPublishRateInBytes(publishRateInBytes);
+            resourcegroup.setReplicationDispatchRateInMsgs(replicationDispatchRateInMsgs);
+            resourcegroup.setReplicationDispatchRateInBytes(replicationDispatchRateInBytes);
 
             getAdmin().resourcegroups().updateResourceGroup(name, resourcegroup);
         }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
index d567d0b3671..88c60963d3f 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopicPolicies.java
@@ -156,6 +156,10 @@ public class CmdTopicPolicies extends CmdBase {
         jcommander.addCommand("set-auto-subscription-creation", new SetAutoSubscriptionCreation());
         jcommander.addCommand("get-auto-subscription-creation", new GetAutoSubscriptionCreation());
         jcommander.addCommand("remove-auto-subscription-creation", new RemoveAutoSubscriptionCreation());
+
+        jcommander.addCommand("set-resource-group", new SetResourceGroup());
+        jcommander.addCommand("get-resource-group", new GetResourceGroup());
+        jcommander.addCommand("remove-resource-group", new RemoveResourceGroup());
     }
 
     @Parameters(commandDescription = "Get entry filters for a topic")
@@ -1931,6 +1935,61 @@ public class CmdTopicPolicies extends CmdBase {
         }
     }
 
+    @Parameters(commandDescription = "Get ResourceGroup for a topic")
+    private class GetResourceGroup extends CliCommand {
+        @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+        private java.util.List<String> params;
+
+        @Parameter(names = {"--applied", "-a"}, description = "Get the applied policy of the topic")
+        private boolean applied = false;
+
+        @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+                + "If set to true, broker returned global topic policies")
+        private boolean isGlobal = false;
+
+        @Override
+        void run() throws PulsarAdminException {
+            String topic = validateTopicName(params);
+            print(getTopicPolicies(isGlobal).getResourceGroup(topic, applied));
+        }
+    }
+
+    @Parameters(commandDescription = "Set ResourceGroup for a topic")
+    private class SetResourceGroup extends CliCommand {
+        @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+        private java.util.List<String> params;
+
+        @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+                + "If set to true, broker returned global topic policies")
+        private boolean isGlobal = false;
+
+        @Parameter(names = {"--resource-group-name", "-rgn"}, description = "ResourceGroup name", required = true)
+        private String rgName;
+
+        @Override
+        void run() throws PulsarAdminException {
+            String topic = validateTopicName(params);
+            getTopicPolicies(isGlobal).setResourceGroup(topic, rgName);
+        }
+    }
+
+    @Parameters(commandDescription = "Remove ResourceGroup from a topic")
+    private class RemoveResourceGroup extends CliCommand {
+        @Parameter(description = "persistent://tenant/namespace/topic", required = true)
+        private java.util.List<String> params;
+
+        @Parameter(names = {"--global", "-g"}, description = "Whether to get this policy globally. "
+                + "If set to true, broker returned global topic policies")
+        private boolean isGlobal = false;
+
+
+        @Override
+        void run() throws PulsarAdminException {
+            String topic = validateTopicName(params);
+            getTopicPolicies(isGlobal).removeResourceGroup(topic);
+        }
+    }
+
     private TopicPolicies getTopicPolicies(boolean isGlobal) {
         return getAdmin().topicPolicies(isGlobal);
     }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
index 0249272b72d..2607e14ffe7 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
@@ -60,6 +60,8 @@ public class HierarchyTopicPolicies {
     final PolicyHierarchyValue<Boolean> schemaValidationEnforced;
     final PolicyHierarchyValue<EntryFilters> entryFilters;
 
+    final PolicyHierarchyValue<String> resourceGroupName;
+
     public HierarchyTopicPolicies() {
         replicationClusters = new PolicyHierarchyValue<>();
         retentionPolicies = new PolicyHierarchyValue<>();
@@ -90,5 +92,6 @@ public class HierarchyTopicPolicies {
         dispatchRate = new PolicyHierarchyValue<>();
         schemaValidationEnforced = new PolicyHierarchyValue<>();
         entryFilters = new PolicyHierarchyValue<>();
+        resourceGroupName = new PolicyHierarchyValue<>();
     }
 }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
index 4a76170d116..2d62d09cfb1 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/TopicPolicies.java
@@ -86,6 +86,8 @@ public class TopicPolicies {
 
     private Boolean schemaValidationEnforced;
 
+    private String resourceGroupName;
+
     public boolean isGlobalPolicies() {
         return isGlobal != null && isGlobal;
     }
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
index 6933f5cc7ed..c744affbb7e 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/ReplicatorStatsImpl.java
@@ -40,6 +40,16 @@ public class ReplicatorStatsImpl implements ReplicatorStats {
     /** Total throughput delivered to the replication-subscriber (bytes/s). */
     public double msgThroughputOut;
 
+    /**
+     * Total bytes delivered to the replication-subscriber (bytes).
+     */
+    public long bytesOutCounter;
+
+    /**
+     * Total messages delivered to the replication-subscriber (msg).
+     */
+    public long msgOutCounter;
+
     /** Total rate of messages expired (msg/s). */
     public double msgRateExpired;