You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/10/01 18:11:09 UTC
(pulsar) 19/38: [fix][broker] Apply subscription policies to namespace and topic subscription operations (#26767)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch lh-branch-4.2-backports-2026-10-01
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 4a4d6f90ada7b5aa03dfca4b95f537267d1eebca
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Wed Sep 30 02:24:05 2026 +0300
[fix][broker] Apply subscription policies to namespace and topic subscription operations (#26767)
(cherry picked from commit 8fdca04d083d385be5938f0b0b3f118bb139bc0b)
---
.../broker/authorization/AuthorizationService.java | 25 +
.../authorization/PulsarAuthorizationProvider.java | 39 +-
.../apache/pulsar/broker/admin/AdminResource.java | 71 ++
.../pulsar/broker/admin/impl/NamespacesBase.java | 234 ++++--
.../broker/admin/impl/PersistentTopicsBase.java | 130 ++-
.../apache/pulsar/broker/admin/v2/Namespaces.java | 11 +-
.../pulsar/broker/web/PulsarWebResource.java | 46 +-
.../org/apache/pulsar/broker/admin/AuthZTest.java | 11 +-
.../pulsar/broker/admin/NamespaceAuthZTest.java | 880 +++++++++++++++++++++
.../NamespaceSubscriptionAnonymousRoleTest.java | 196 +++++
.../admin/NamespaceSubscriptionProxyRoleTest.java | 157 ++++
.../apache/pulsar/broker/admin/NamespacesTest.java | 2 +-
.../MultiRolesTokenAuthorizationProviderTest.java | 237 ++++++
.../client/admin/internal/NamespacesImpl.java | 14 +
14 files changed, 1943 insertions(+), 110 deletions(-)
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
index d3d522386884..021b05b31c53 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java
@@ -720,6 +720,31 @@ public class AuthorizationService {
}
}
+ /**
+ * Checks a namespace operation, using separate authentication data for the original principal of a proxied
+ * request.
+ */
+ public CompletableFuture<Boolean> allowNamespaceOperationAsync(NamespaceName namespaceName,
+ NamespaceOperation operation,
+ String originalRole,
+ String role,
+ AuthenticationDataSource originalAuthData,
+ AuthenticationDataSource authData) {
+ if (!isValidOriginalPrincipal(role, originalRole, authData)) {
+ return CompletableFuture.completedFuture(false);
+ }
+ if (isProxyRole(role) && !isWebsocketPrinciple(originalRole)) {
+ CompletableFuture<Boolean> isRoleAuthorizedFuture = allowNamespaceOperationAsync(
+ namespaceName, operation, role, authData);
+ CompletableFuture<Boolean> isOriginalAuthorizedFuture = allowNamespaceOperationAsync(
+ namespaceName, operation, originalRole, originalAuthData);
+ return isRoleAuthorizedFuture.thenCombine(isOriginalAuthorizedFuture,
+ (isRoleAuthorized, isOriginalAuthorized) -> isRoleAuthorized && isOriginalAuthorized);
+ } else {
+ return allowNamespaceOperationAsync(namespaceName, operation, role, authData);
+ }
+ }
+
/**
* Grant authorization-action permission on a namespace to the given client.
*
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
index 05a30bd8ac2c..5804912facec 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.broker.authorization;
import static java.util.Objects.requireNonNull;
+import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import com.google.common.collect.Sets;
import java.io.IOException;
@@ -44,8 +45,10 @@ import org.apache.pulsar.common.policies.data.AuthPolicies;
import org.apache.pulsar.common.policies.data.BrokerOperation;
import org.apache.pulsar.common.policies.data.ClusterOperation;
import org.apache.pulsar.common.policies.data.NamespaceOperation;
+import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.PolicyName;
import org.apache.pulsar.common.policies.data.PolicyOperation;
+import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
import org.apache.pulsar.common.policies.data.TenantOperation;
import org.apache.pulsar.common.policies.data.TopicOperation;
import org.apache.pulsar.common.util.FutureUtil;
@@ -207,6 +210,39 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
});
}
+ /**
+ * Applies the namespace subscription policies, as {@link #canConsumeAsync} does for topic operations.
+ */
+ private CompletableFuture<Boolean> allowNamespaceSubscriptionOpsAsync(NamespaceName namespaceName, String role,
+ AuthenticationDataSource authData,
+ NamespaceOperation operation) {
+ final String subscription = authData != null ? authData.getSubscription() : null;
+ if (isBlank(subscription)) {
+ return allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.consume);
+ }
+ return pulsarResources.getNamespaceResources().getPoliciesAsync(namespaceName).thenCompose(policies -> {
+ if (policies.isPresent() && !isSubscriptionAllowed(policies.get(), role, subscription)) {
+ if (log.isDebugEnabled()) {
+ log.debug("Role {} is not allowed to access the subscription {} of namespace {} for operation {}",
+ role, subscription, namespaceName, operation);
+ }
+ return CompletableFuture.completedFuture(false);
+ }
+ return allowTheSpecifiedActionOpsAsync(namespaceName, role, authData, AuthAction.consume);
+ });
+ }
+
+ private static boolean isSubscriptionAllowed(Policies policies, String role, String subscription) {
+ Set<String> roles = policies.auth_policies.getSubscriptionAuthentication().get(subscription);
+ if (roles != null && !roles.isEmpty() && !roles.contains(role)) {
+ return false;
+ }
+ if (policies.subscription_auth_mode == SubscriptionAuthMode.Prefix) {
+ return role != null && subscription.startsWith(role);
+ }
+ return true;
+ }
+
private CompletableFuture<Boolean> allowTheSpecifiedActionOpsAsync(NamespaceName namespaceName, String role,
AuthenticationDataSource authenticationData,
AuthAction authAction) {
@@ -602,8 +638,9 @@ public class PulsarAuthorizationProvider implements AuthorizationProvider {
case GET_BUNDLE:
return allowConsumeOrProduceOpsAsync(namespaceName, role, authData);
case UNSUBSCRIBE:
- case TRIM_TOPIC:
case CLEAR_BACKLOG:
+ return allowNamespaceSubscriptionOpsAsync(namespaceName, role, authData, operation);
+ case TRIM_TOPIC:
return allowTheSpecifiedActionOpsAsync(
namespaceName, role, authData, AuthAction.consume);
case CREATE_TOPIC:
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index 3147555d56ea..b7e2ef398ebf 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -44,7 +44,9 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.authorization.AuthorizationService;
import org.apache.pulsar.broker.namespace.TopicExistsInfo;
import org.apache.pulsar.broker.resources.ClusterResources;
@@ -970,6 +972,75 @@ public abstract class AdminResource extends PulsarWebResource {
return !isRedirectException(ex) && !is4xxRestException(ex);
}
+ /**
+ * Whether the caller is a super user or an admin of the tenant; true when authorization is disabled.
+ */
+ protected CompletableFuture<Boolean> isSuperUserOrTenantAdminAsync() {
+ if (!pulsar().getConfiguration().isAuthenticationEnabled()
+ || !pulsar().getBrokerService().isAuthorizationEnabled()) {
+ return CompletableFuture.completedFuture(true);
+ }
+ AuthorizationService authorizationService = pulsar().getBrokerService().getAuthorizationService();
+ String role = clientAppId();
+ String originalRole = originalPrincipal();
+ if (!authorizationService.isValidOriginalPrincipal(role, originalRole, clientAuthData())) {
+ return CompletableFuture.completedFuture(false);
+ }
+ CompletableFuture<Boolean> isAdmin = isSuperUserOrTenantAdminAsync(role, clientAuthData());
+ if (authorizationService.isProxyRole(role) && !authorizationService.isWebsocketPrinciple(originalRole)) {
+ // the original principal is checked with its own auth data, not with the proxy's
+ isAdmin = isAdmin.thenCombine(isSuperUserOrTenantAdminAsync(originalRole, originalPrincipalAuthData()),
+ (isRoleAdmin, isOriginalAdmin) -> isRoleAdmin && isOriginalAdmin);
+ }
+ return isAdmin;
+ }
+
+ private CompletableFuture<Boolean> isSuperUserOrTenantAdminAsync(String role, AuthenticationDataSource authData) {
+ return pulsar().getBrokerService().getAuthorizationService()
+ .isSuperUserOrTenantAdmin(namespaceName.getTenant(), role, authData)
+ .exceptionally(ex -> {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Tenant admin check failed for role {}", namespaceName, role, ex);
+ }
+ return false;
+ });
+ }
+
+ /**
+ * Maps an authorization check to false when it is rejected.
+ */
+ protected static CompletableFuture<Boolean> isAuthorizedAsync(CompletableFuture<Void> authorizationCheck) {
+ return authorizationCheck.handle((__, ex) -> {
+ if (ex == null) {
+ return true;
+ }
+ Throwable cause = FutureUtil.unwrapCompletionException(ex);
+ if (cause instanceof WebApplicationException wae
+ && (wae.getResponse().getStatus() == Status.FORBIDDEN.getStatusCode()
+ || wae.getResponse().getStatus() == Status.UNAUTHORIZED.getStatusCode())) {
+ return false;
+ }
+ // PulsarAuthorizationProvider rejects a subscription outside the role prefix with this exception
+ if (cause instanceof PulsarServerException) {
+ return false;
+ }
+ throw FutureUtil.wrapToCompletionException(cause);
+ });
+ }
+
+ /**
+ * Completes with {@code defaultValue} when an admin client call fails with "not found".
+ */
+ protected static <T> CompletableFuture<T> ignoreNotFound(CompletableFuture<T> future, T defaultValue) {
+ return future.exceptionally(ex -> {
+ Throwable cause = FutureUtil.unwrapCompletionException(ex);
+ if (cause instanceof PulsarAdminException.NotFoundException) {
+ return defaultValue;
+ }
+ throw FutureUtil.wrapToCompletionException(cause);
+ });
+ }
+
protected static String getTopicNotFoundErrorMessage(String topic) {
return String.format("Topic %s not found", topic);
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index 653cb39ca910..7e5ece48340d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -36,6 +36,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -64,14 +65,17 @@ import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyE
import org.apache.pulsar.broker.service.Subscription;
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
+import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.broker.topiclistlimit.TopicListMemoryLimiter;
import org.apache.pulsar.broker.topiclistlimit.TopicListSizeResultCache;
import org.apache.pulsar.broker.web.RestException;
import org.apache.pulsar.client.admin.GrantTopicPermissionOptions;
+import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.RevokeTopicPermissionOptions;
+import org.apache.pulsar.client.admin.internal.NamespacesImpl;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace;
import org.apache.pulsar.common.naming.NamedEntity;
@@ -1825,20 +1829,28 @@ public abstract class NamespacesBase extends AdminResource {
protected CompletableFuture<Void> internalClearNamespaceBacklogAsync(boolean authoritative) {
return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG)
- .thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory()
- .getBundlesAsync(namespaceName))
- .thenCompose(bundles -> {
- final List<CompletableFuture<Void>> futures = new ArrayList<>();
- for (NamespaceBundle nsBundle : bundles.getBundles()) {
- try {
- futures.add(pulsar().getAdminClient().namespaces()
- .clearNamespaceBundleBacklogAsync(namespaceName.toString(),
- nsBundle.getBundleRange()));
- } catch (PulsarServerException e) {
- return CompletableFuture.failedFuture(e);
- }
+ .thenCompose(__ -> isSuperUserOrTenantAdminAsync())
+ .thenCompose(isAdmin -> {
+ if (!isAdmin) {
+ // the topics are listed from local metadata: redirect if a peer cluster owns the namespace
+ return validateGlobalNamespaceOwnershipAsync(namespaceName)
+ .thenCompose(__ -> clearAuthorizedSubscriptionsBacklogAsync());
}
- return FutureUtil.waitForAll(futures);
+ return pulsar().getNamespaceService().getNamespaceBundleFactory()
+ .getBundlesAsync(namespaceName)
+ .thenCompose(bundles -> {
+ final List<CompletableFuture<Void>> futures = new ArrayList<>();
+ for (NamespaceBundle nsBundle : bundles.getBundles()) {
+ try {
+ futures.add(pulsar().getAdminClient().namespaces()
+ .clearNamespaceBundleBacklogAsync(namespaceName.toString(),
+ nsBundle.getBundleRange()));
+ } catch (PulsarServerException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
+ return FutureUtil.waitForAll(futures);
+ });
}).thenRun(() -> log.info("[{}] Successfully cleared backlog on all the bundles for namespace {}",
clientAppId(), namespaceName));
}
@@ -1860,7 +1872,8 @@ public abstract class NamespacesBase extends AdminResource {
// even if not loaded.
validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange,
authoritative, false))
- .thenCompose(bundle -> clearBacklogAsync(bundle, null))
+ .thenCompose(bundle -> isSuperUserOrTenantAdminAsync()
+ .thenCompose(isAdmin -> clearBacklogAsync(bundle, null, isAdmin)))
.thenRun(() -> log.info("[{}] Successfully cleared backlog on namespace bundle {}/{}", clientAppId(),
namespaceName, bundleRange));
}
@@ -1871,30 +1884,35 @@ public abstract class NamespacesBase extends AdminResource {
return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null"));
}
- return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG)
- .thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory()
- .getBundlesAsync(namespaceName))
- .thenCompose(bundles -> {
- final List<CompletableFuture<Void>> futures = new ArrayList<>();
- for (NamespaceBundle nsBundle : bundles.getBundles()) {
- try {
- futures.add(pulsar().getAdminClient().namespaces()
- .clearNamespaceBundleBacklogForSubscriptionAsync(
- namespaceName.toString(), nsBundle.getBundleRange(), subscription));
- } catch (PulsarServerException e) {
- return CompletableFuture.failedFuture(e);
- }
+ return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG, subscription)
+ .thenCompose(__ -> isSuperUserOrTenantAdminAsync())
+ .thenCompose(canClearReplicatorBacklog -> {
+ if (!canClearReplicatorBacklog && isReplicatorCursorName(subscription)) {
+ return CompletableFuture.failedFuture(replicatorCursorForbidden());
}
- return FutureUtil.waitForAll(futures);
+ return pulsar().getNamespaceService().getNamespaceBundleFactory()
+ .getBundlesAsync(namespaceName)
+ .thenCompose(bundles -> {
+ final List<CompletableFuture<Void>> futures = new ArrayList<>();
+ for (NamespaceBundle nsBundle : bundles.getBundles()) {
+ try {
+ futures.add(clearNamespaceBundleBacklogForSubscriptionAsync(
+ nsBundle.getBundleRange(), subscription,
+ !canClearReplicatorBacklog));
+ } catch (PulsarServerException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ }
+ return FutureUtil.waitForAll(futures);
+ });
}).thenRun(() -> log.info(
"[{}] Successfully cleared backlog for subscription {} on all the bundles for namespace {}",
clientAppId(), subscription, namespaceName));
}
@SuppressWarnings("deprecation")
- protected CompletableFuture<Void> internalClearNamespaceBundleBacklogForSubscriptionAsync(String subscription,
- String bundleRange,
- boolean authoritative) {
+ protected CompletableFuture<Void> internalClearNamespaceBundleBacklogForSubscriptionAsync(
+ String subscription, String bundleRange, boolean authoritative, boolean subscriptionOnly) {
if (subscription == null) {
return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null"));
}
@@ -1902,7 +1920,7 @@ public abstract class NamespacesBase extends AdminResource {
return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "BundleRange should not be null"));
}
- return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG)
+ return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG, subscription)
.thenCompose(__ -> {
// check cluster ownership for a given global namespace: redirect if peer-cluster owns it
return validateGlobalNamespaceOwnershipAsync(namespaceName);
@@ -1912,7 +1930,11 @@ public abstract class NamespacesBase extends AdminResource {
// even if not loaded.
validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange,
authoritative, false))
- .thenCompose(bundle -> clearBacklogAsync(bundle, subscription))
+ .thenCompose(bundle -> (subscriptionOnly
+ ? CompletableFuture.completedFuture(false)
+ : isSuperUserOrTenantAdminAsync())
+ .thenCompose(canClearReplicatorBacklog ->
+ clearBacklogAsync(bundle, subscription, canClearReplicatorBacklog)))
.thenRun(() -> log.info(
"[{}] Successfully cleared backlog for subscription {} on namespace bundle {}/{}",
clientAppId(), subscription, namespaceName, bundleRange));
@@ -1924,7 +1946,7 @@ public abstract class NamespacesBase extends AdminResource {
return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "Subscription should not be null"));
}
- return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE)
+ return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE, subscription)
.thenCompose(__ -> pulsar().getNamespaceService().getNamespaceBundleFactory()
.getBundlesAsync(namespaceName))
.thenCompose(bundles -> {
@@ -1952,7 +1974,7 @@ public abstract class NamespacesBase extends AdminResource {
return FutureUtil.failedFuture(new RestException(Status.BAD_REQUEST, "BundleRange should not be null"));
}
- return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE)
+ return validateNamespaceOperationAsync(namespaceName, NamespaceOperation.UNSUBSCRIBE, subscription)
.thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName))
.thenCompose(__ ->
validateNamespaceBundleOwnershipAsync(namespaceName, bundleRange,
@@ -2130,35 +2152,135 @@ public abstract class NamespacesBase extends AdminResource {
return checkBacklogQuota(quota, retention);
}
- private CompletableFuture<Void> clearBacklogAsync(NamespaceBundle bundle, String subscription) {
+ /**
+ * Whether the caller may clear the backlog of the subscription, checked as for a single subscription.
+ */
+ private CompletableFuture<Boolean> canClearSubscriptionBacklogAsync(String subscription) {
+ if (isReplicatorCursorName(subscription)) {
+ return CompletableFuture.completedFuture(false);
+ }
+ return isAuthorizedAsync(validateNamespaceOperationAsync(namespaceName, NamespaceOperation.CLEAR_BACKLOG,
+ subscription));
+ }
+
+ /**
+ * Clears the backlog of the namespace subscriptions that the caller may clear, without replicators. Dispatched
+ * per topic, because bundle requests are made with the broker's identity.
+ */
+ private CompletableFuture<Void> clearAuthorizedSubscriptionsBacklogAsync() {
+ final PulsarAdmin admin;
+ try {
+ admin = pulsar().getAdminClient();
+ } catch (PulsarServerException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ final Map<String, CompletableFuture<Boolean>> allowed = new ConcurrentHashMap<>();
+ return pulsar().getNamespaceService().getListOfPersistentTopics(namespaceName)
+ .thenCompose(topics -> FutureUtil.waitForAll(topics.stream()
+ .filter(topic -> !pulsar().getBrokerService().isSystemTopic(TopicName.get(topic)))
+ .map(topic -> ignoreNotFound(admin.topics().getSubscriptionsAsync(topic), List.<String>of())
+ .thenCompose(subscriptions -> FutureUtil.waitForAll(subscriptions.stream()
+ .map(sub -> allowed.computeIfAbsent(sub,
+ this::canClearSubscriptionBacklogAsync)
+ .thenCompose(canClear -> canClear
+ ? ignoreNotFound(
+ admin.topics().skipAllMessagesAsync(topic, sub), null)
+ : CompletableFuture.<Void>completedFuture(null)))
+ .toList())))
+ .toList()));
+ }
+
+ private boolean isReplicatorCursorName(String subscription) {
+ return subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix() + ".");
+ }
+
+ private static RestException replicatorCursorForbidden() {
+ return new RestException(Status.FORBIDDEN,
+ "Clearing the backlog of a replicator cursor is not allowed for this request");
+ }
+
+ private CompletableFuture<Void> clearNamespaceBundleBacklogForSubscriptionAsync(String bundleRange,
+ String subscription,
+ boolean subscriptionOnly)
+ throws PulsarServerException {
+ Namespaces namespaces = pulsar().getAdminClient().namespaces();
+ if (!subscriptionOnly) {
+ return namespaces.clearNamespaceBundleBacklogForSubscriptionAsync(namespaceName.toString(),
+ bundleRange, subscription);
+ }
+ if (namespaces instanceof NamespacesImpl namespacesImpl) {
+ return namespacesImpl.clearNamespaceBundleBacklogForSubscriptionAsync(namespaceName.toString(),
+ bundleRange, subscription, true);
+ }
+ return CompletableFuture.failedFuture(new RestException(Status.INTERNAL_SERVER_ERROR,
+ "Unsupported admin client implementation " + namespaces.getClass().getName()));
+ }
+
+ /**
+ * When {@code canClearReplicatorBacklog} is false, only subscriptions can be cleared, and without a
+ * subscription name only those that the caller may clear.
+ */
+ private CompletableFuture<Void> clearBacklogAsync(NamespaceBundle bundle, String subscription,
+ boolean canClearReplicatorBacklog) {
+ final String replicatorPrefix = pulsar().getConfiguration().getReplicatorPrefix();
+ if (subscription != null && !canClearReplicatorBacklog && isReplicatorCursorName(subscription)) {
+ return CompletableFuture.failedFuture(replicatorCursorForbidden());
+ }
return pulsar().getNamespaceService().getOwnedPersistentTopicListForNamespaceBundle(bundle)
.thenCompose(topicsInBundle -> {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
- final String replicatorPrefix = pulsar().getConfiguration().getReplicatorPrefix();
- final String finalSubscription = subscription == null ? null
- : PersistentReplicator.getRemoteCluster(replicatorPrefix, subscription)
- .orElse(subscription);
-
+ List<CompletableFuture<Optional<Topic>>> topicFutures = new ArrayList<>();
for (String topic : topicsInBundle) {
TopicName topicName = TopicName.get(topic);
if (pulsar().getBrokerService().isSystemTopic(topicName)) {
continue;
}
- futures.add(pulsar().getBrokerService().getTopic(topicName.toString(), false)
- .thenCompose(optTopic -> {
- if (optTopic.isEmpty()) {
- return CompletableFuture.completedFuture(null);
- }
- Topic loaded = optTopic.get();
- if (!(loaded instanceof PersistentTopic persistentTopic)) {
- return CompletableFuture.completedFuture(null);
- }
- return finalSubscription != null
- ? persistentTopic.clearBacklog(finalSubscription)
- : persistentTopic.clearBacklog();
- }));
+ topicFutures.add(pulsar().getBrokerService().getTopic(topicName.toString(), false));
+ }
+ return FutureUtil.waitForAll(topicFutures).thenApply(__ -> topicFutures.stream()
+ .map(CompletableFuture::join)
+ .flatMap(Optional::stream)
+ .filter(PersistentTopic.class::isInstance)
+ .map(PersistentTopic.class::cast)
+ .toList());
+ })
+ .thenCompose(persistentTopics -> {
+ if (subscription == null) {
+ if (canClearReplicatorBacklog) {
+ return FutureUtil.waitForAll(persistentTopics.stream()
+ .map(PersistentTopic::clearBacklog)
+ .toList());
+ }
+ // only the subscriptions that the caller may clear
+ Map<String, CompletableFuture<Boolean>> allowed = new HashMap<>();
+ List<CompletableFuture<Void>> futures = new ArrayList<>();
+ for (PersistentTopic topic : persistentTopics) {
+ topic.getSubscriptions().forEach((name, sub) -> futures.add(
+ allowed.computeIfAbsent(name, this::canClearSubscriptionBacklogAsync)
+ .thenCompose(canClear -> canClear ? sub.clearBacklog()
+ : CompletableFuture.completedFuture(null))));
+ }
+ return FutureUtil.waitForAll(futures);
+ }
+ if (canClearReplicatorBacklog) {
+ final String cursorName = PersistentReplicator.getRemoteCluster(replicatorPrefix,
+ subscription).orElse(subscription);
+ return FutureUtil.waitForAll(persistentTopics.stream()
+ .map(topic -> topic.clearBacklog(cursorName))
+ .toList());
+ }
+ for (PersistentTopic topic : persistentTopics) {
+ if (topic.getSubscription(subscription) == null
+ && (topic.getPersistentReplicator(subscription) != null
+ || topic.getShadowReplicators().containsKey(subscription))) {
+ return CompletableFuture.failedFuture(replicatorCursorForbidden());
+ }
+ }
+ List<CompletableFuture<Void>> futures = new ArrayList<>();
+ for (PersistentTopic topic : persistentTopics) {
+ PersistentSubscription sub = topic.getSubscription(subscription);
+ futures.add(sub != null ? sub.clearBacklog()
+ : FutureUtil.failedFuture(new BrokerServiceException("Cursor not found")));
}
-
return FutureUtil.waitForAll(futures);
}).exceptionally(ex -> {
Throwable cause = FutureUtil.unwrapCompletionException(ex);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index cc902b3ee37b..f1c3056f2bc8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -1946,30 +1946,38 @@ public class PersistentTopicsBase extends AdminResource {
boolean authoritative) {
validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES)
.thenCompose(__ -> validateGlobalNamespaceOwnershipAsync(namespaceName))
- .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false))
- .thenAccept(partitionMetadata -> {
+ .thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)
+ .thenCombine(isSuperUserOrTenantAdminAsync(), Pair::of))
+ .thenAccept(metadataAndIsAdmin -> {
+ final PartitionedTopicMetadata partitionMetadata = metadataAndIsAdmin.getLeft();
+ final boolean isAdmin = metadataAndIsAdmin.getRight();
if (topicName.isPartitioned()) {
internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse,
- partitionMetadata, expireTimeInSeconds, authoritative);
+ partitionMetadata, expireTimeInSeconds, authoritative, !isAdmin);
} else {
if (partitionMetadata.partitions > 0) {
final List<CompletableFuture<Void>> futures = new ArrayList<>(partitionMetadata.partitions);
- // expire messages for each partition topic
- for (int i = 0; i < partitionMetadata.partitions; i++) {
- TopicName topicNamePartition = topicName.getPartition(i);
- try {
- futures.add(pulsar()
- .getAdminClient()
- .topics()
- .expireMessagesForAllSubscriptionsAsync(
- topicNamePartition.toString(), expireTimeInSeconds));
- } catch (Exception e) {
- log.error("[{}] Failed to expire messages up to {} on {}",
- clientAppId(), expireTimeInSeconds,
- topicNamePartition, e);
- asyncResponse.resume(new RestException(e));
- return;
+ if (!isAdmin) {
+ futures.add(expireMessagesForAllowedSubscriptionsOfPartitionsAsync(
+ partitionMetadata.partitions, expireTimeInSeconds));
+ } else {
+ // expire messages for each partition topic
+ for (int i = 0; i < partitionMetadata.partitions; i++) {
+ TopicName topicNamePartition = topicName.getPartition(i);
+ try {
+ futures.add(pulsar()
+ .getAdminClient()
+ .topics()
+ .expireMessagesForAllSubscriptionsAsync(
+ topicNamePartition.toString(), expireTimeInSeconds));
+ } catch (Exception e) {
+ log.error("[{}] Failed to expire messages up to {} on {}",
+ clientAppId(), expireTimeInSeconds,
+ topicNamePartition, e);
+ asyncResponse.resume(new RestException(e));
+ return;
+ }
}
}
@@ -1991,7 +1999,7 @@ public class PersistentTopicsBase extends AdminResource {
});
} else {
internalExpireMessagesForAllSubscriptionsForNonPartitionedTopic(asyncResponse,
- partitionMetadata, expireTimeInSeconds, authoritative);
+ partitionMetadata, expireTimeInSeconds, authoritative, !isAdmin);
}
}
}
@@ -2011,7 +2019,8 @@ public class PersistentTopicsBase extends AdminResource {
PartitionedTopicMetadata
partitionMetadata,
int expireTimeInSeconds,
- boolean authoritative) {
+ boolean authoritative,
+ boolean onlyAllowedSubscriptions) {
// validate ownership and redirect if current broker is not owner
validateTopicOwnershipAsync(topicName, authoritative)
.thenCompose(__ -> getTopicReferenceAsync(topicName).thenAccept(t -> {
@@ -2026,25 +2035,15 @@ public class PersistentTopicsBase extends AdminResource {
return;
}
PersistentTopic topic = (PersistentTopic) t;
- final List<CompletableFuture<Void>> futures =
- new ArrayList<>((int) topic.getReplicators().size());
- List<String> subNames =
- new ArrayList<>((int) topic.getSubscriptions().size());
- subNames.addAll(topic.getSubscriptions().keySet().stream().filter(
- subName -> !subName.equals(Compactor.COMPACTION_SUBSCRIPTION)).toList());
- for (int i = 0; i < subNames.size(); i++) {
- try {
- futures.add(internalExpireMessagesByTimestampForSinglePartitionAsync(partitionMetadata,
- subNames.get(i), expireTimeInSeconds));
- } catch (Exception e) {
- log.error("[{}] Failed to expire messages for all subscription up to {} on {}",
- clientAppId(), expireTimeInSeconds, topicName, e);
- asyncResponse.resume(new RestException(e));
- return;
- }
- }
-
- FutureUtil.waitForAll(futures).handle((result, exception) -> {
+ List<String> subNames = topic.getSubscriptions().keySet().stream().filter(
+ subName -> !subName.equals(Compactor.COMPACTION_SUBSCRIPTION)).toList();
+ CompletableFuture<List<String>> subNamesFuture = onlyAllowedSubscriptions
+ ? filterSubscriptionsAllowedToExpireAsync(subNames)
+ : CompletableFuture.completedFuture(subNames);
+ subNamesFuture.thenCompose(names -> FutureUtil.waitForAll(names.stream()
+ .map(subName -> internalExpireMessagesByTimestampForSinglePartitionAsync(
+ partitionMetadata, subName, expireTimeInSeconds))
+ .toList())).handle((result, exception) -> {
if (exception != null) {
Throwable throwable = FutureUtil.unwrapCompletionException(exception);
if (throwable instanceof RestException) {
@@ -2072,6 +2071,59 @@ public class PersistentTopicsBase extends AdminResource {
});
}
+ /**
+ * Whether the caller may expire messages of the subscription, checked as for a single subscription.
+ */
+ private CompletableFuture<Boolean> canExpireSubscriptionAsync(String subName) {
+ if (AbstractReplicator.getRemoteCluster(pulsar().getConfiguration().getReplicatorPrefix(), subName)
+ .isPresent()) {
+ return CompletableFuture.completedFuture(false);
+ }
+ return isAuthorizedAsync(validateTopicOperationAsync(topicName, TopicOperation.EXPIRE_MESSAGES, subName));
+ }
+
+ private CompletableFuture<List<String>> filterSubscriptionsAllowedToExpireAsync(List<String> subNames) {
+ List<CompletableFuture<Boolean>> checks = subNames.stream().map(this::canExpireSubscriptionAsync).toList();
+ return FutureUtil.waitForAll(checks).thenApply(__ -> {
+ List<String> allowed = new ArrayList<>();
+ for (int i = 0; i < subNames.size(); i++) {
+ if (checks.get(i).join()) {
+ allowed.add(subNames.get(i));
+ }
+ }
+ return allowed;
+ });
+ }
+
+ /**
+ * Expires messages of the partition subscriptions that the caller may expire. Dispatched per subscription,
+ * because partition requests are made with the broker's identity.
+ */
+ private CompletableFuture<Void> expireMessagesForAllowedSubscriptionsOfPartitionsAsync(int partitions,
+ int expireTimeInSeconds) {
+ final PulsarAdmin admin;
+ try {
+ admin = pulsar().getAdminClient();
+ } catch (PulsarServerException e) {
+ return CompletableFuture.failedFuture(e);
+ }
+ final Map<String, CompletableFuture<Boolean>> allowed = new ConcurrentHashMap<>();
+ final List<CompletableFuture<Void>> futures = new ArrayList<>(partitions);
+ for (int i = 0; i < partitions; i++) {
+ final String partition = topicName.getPartition(i).toString();
+ futures.add(ignoreNotFound(admin.topics().getSubscriptionsAsync(partition), List.<String>of())
+ .thenCompose(subNames -> FutureUtil.waitForAll(subNames.stream()
+ .filter(subName -> !subName.equals(Compactor.COMPACTION_SUBSCRIPTION))
+ .map(subName -> allowed.computeIfAbsent(subName, this::canExpireSubscriptionAsync)
+ .thenCompose(canExpire -> canExpire
+ ? ignoreNotFound(admin.topics().expireMessagesAsync(partition, subName,
+ expireTimeInSeconds), null)
+ : CompletableFuture.<Void>completedFuture(null)))
+ .toList())));
+ }
+ return FutureUtil.waitForAll(futures);
+ }
+
protected CompletableFuture<Void> internalResetCursorAsync(String subName, long timestamp,
boolean authoritative) {
return validateTopicOperationAsync(topicName, TopicOperation.RESET_CURSOR, subName)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index ad01dd6f39c1..2146c5db7cc8 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -1520,7 +1520,9 @@ public class Namespaces extends NamespacesBase {
internalClearNamespaceBacklogAsync(authoritative)
.thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
.exceptionally(ex -> {
- log.error("[{}] Failed to clear backlog on namespace {}", clientAppId(), namespaceName, ex);
+ if (!isRedirectException(ex)) {
+ log.error("[{}] Failed to clear backlog on namespace {}", clientAppId(), namespaceName, ex);
+ }
resumeAsyncResponseExceptionally(asyncResponse, ex);
return null;
});
@@ -1583,9 +1585,12 @@ public class Namespaces extends NamespacesBase {
@PathParam("tenant") String tenant,
@PathParam("namespace") String namespace, @PathParam("subscription") String subscription,
@PathParam("bundle") String bundleRange,
- @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
+ @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
+ @ApiParam(hidden = true) @QueryParam("subscriptionOnly") @DefaultValue("false")
+ boolean subscriptionOnly) {
validateNamespaceName(tenant, namespace);
- internalClearNamespaceBundleBacklogForSubscriptionAsync(subscription, bundleRange, authoritative)
+ internalClearNamespaceBundleBacklogForSubscriptionAsync(subscription, bundleRange, authoritative,
+ subscriptionOnly)
.thenAccept(__ -> asyncResponse.resume(Response.noContent().build()))
.exceptionally(ex -> {
log.error("[{}] Failed to clear backlog for subscription {} on namespace bundle {}/{}",
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
index 7bec18341b0b..afe0a90b6167 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java
@@ -53,6 +53,7 @@ import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.authentication.AuthenticationDataForwarded;
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
+import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription;
import org.apache.pulsar.broker.authentication.AuthenticationParameters;
import org.apache.pulsar.broker.authorization.AuthorizationService;
import org.apache.pulsar.broker.loadbalance.LoadManager;
@@ -1024,6 +1025,15 @@ public abstract class PulsarWebResource {
public CompletableFuture<Void> validateNamespaceOperationAsync(NamespaceName namespaceName,
NamespaceOperation operation) {
+ return validateNamespaceOperationAsync(namespaceName, operation, null);
+ }
+
+ /**
+ * Validates a namespace operation for a single subscription, passing it to the authorization provider.
+ */
+ public CompletableFuture<Void> validateNamespaceOperationAsync(NamespaceName namespaceName,
+ NamespaceOperation operation,
+ String subscription) {
if (pulsar().getConfiguration().isAuthenticationEnabled()
&& pulsar().getBrokerService().isAuthorizationEnabled()) {
if (!isClientAuthenticated(clientAppId())) {
@@ -1031,9 +1041,21 @@ public abstract class PulsarWebResource {
new RestException(Status.FORBIDDEN, "Need to authenticate to perform the request"));
}
- return pulsar().getBrokerService().getAuthorizationService()
- .allowNamespaceOperationAsync(namespaceName, operation, originalPrincipal(),
- clientAppId(), clientAuthData())
+ AuthorizationService authorizationService = pulsar().getBrokerService().getAuthorizationService();
+ CompletableFuture<Boolean> allowFuture;
+ if (subscription != null) {
+ // Wrap the request auth data so that the subscription is available for every kind of
+ // authentication data, including the auth data of the anonymous role. The original principal
+ // is checked with its own auth data, so that it is not evaluated with the proxy credentials.
+ allowFuture = authorizationService.allowNamespaceOperationAsync(namespaceName, operation,
+ originalPrincipal(), clientAppId(),
+ new AuthenticationDataSubscription(originalPrincipalAuthData(), subscription),
+ new AuthenticationDataSubscription(clientAuthData(), subscription));
+ } else {
+ allowFuture = authorizationService.allowNamespaceOperationAsync(namespaceName, operation,
+ originalPrincipal(), clientAppId(), clientAuthData());
+ }
+ return allowFuture
.thenAccept(isAuthorized -> {
if (!isAuthorized) {
throw new RestException(Status.FORBIDDEN,
@@ -1274,10 +1296,20 @@ public abstract class PulsarWebResource {
new RestException(Status.UNAUTHORIZED, "Need to authenticate to perform the request"));
}
- AuthenticationDataSource authData = clientAuthData();
- authData.setSubscription(subscription);
- return pulsar().getBrokerService().getAuthorizationService()
- .allowTopicOperationAsync(topicName, operation, originalPrincipal(), clientAppId(), authData)
+ AuthorizationService authorizationService = pulsar().getBrokerService().getAuthorizationService();
+ CompletableFuture<Boolean> allowFuture;
+ if (subscription != null) {
+ // Wrap instead of setting the subscription, which some auth data types (e.g. anonymous) ignore.
+ // As for namespace operations, the original principal is checked with its own auth data.
+ allowFuture = authorizationService.allowTopicOperationAsync(topicName, operation,
+ originalPrincipal(), clientAppId(),
+ new AuthenticationDataSubscription(originalPrincipalAuthData(), subscription),
+ new AuthenticationDataSubscription(clientAuthData(), subscription));
+ } else {
+ allowFuture = authorizationService.allowTopicOperationAsync(topicName, operation,
+ originalPrincipal(), clientAppId(), clientAuthData());
+ }
+ return allowFuture
.thenAccept(isAuthorized -> {
if (!isAuthorized) {
throw new RestException(Status.UNAUTHORIZED, String.format(
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java
index 4eb79fb8c5c2..997bbf59beb0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AuthZTest.java
@@ -33,6 +33,7 @@ import org.apache.pulsar.common.policies.data.TopicOperation;
import org.apache.pulsar.security.MockedPulsarStandalone;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@@ -76,15 +77,19 @@ public abstract class AuthZTest extends MockedPulsarStandalone {
getPulsarService().getBrokerService().getAuthorizationService());
FieldUtils.writeField(getPulsarService().getBrokerService(), "authorizationService",
authorizationService, true);
- Mockito.doAnswer(invocationOnMock -> {
+ Answer<Object> allowTopicOperationAsyncAnswer = invocationOnMock -> {
Consumer<InvocationOnMock> localAllowTopicOperationAsyncHandler =
allowTopicOperationAsyncHandler;
if (localAllowTopicOperationAsyncHandler != null) {
localAllowTopicOperationAsyncHandler.accept(invocationOnMock);
}
return invocationOnMock.callRealMethod();
- }).when(authorizationService).allowTopicOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(),
- Mockito.any(), Mockito.any());
+ };
+ Mockito.doAnswer(allowTopicOperationAsyncAnswer).when(authorizationService).allowTopicOperationAsync(
+ Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
+ // used for operations on a subscription
+ Mockito.doAnswer(allowTopicOperationAsyncAnswer).when(authorizationService).allowTopicOperationAsync(
+ Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
doReturn(true)
.when(authorizationService).isValidOriginalPrincipal(Mockito.any(), Mockito.any(), Mockito.any());
Mockito.doAnswer(invocationOnMock -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java
index aadcb8748645..edd575736d98 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceAuthZTest.java
@@ -22,21 +22,29 @@ package org.apache.pulsar.broker.admin;
import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.deleteNamespaceWithRetry;
import static org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy.AutoUpdateDisabled;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import com.google.common.collect.Sets;
import io.jsonwebtoken.Jwts;
import java.io.File;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
+import java.util.function.Supplier;
import lombok.Cleanup;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
@@ -44,8 +52,11 @@ import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.pulsar.broker.BrokerTestUtil;
import org.apache.pulsar.broker.authorization.AuthorizationService;
import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.broker.service.persistent.PersistentReplicator;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
@@ -58,6 +69,7 @@ import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
import org.apache.pulsar.common.policies.data.BacklogQuota;
import org.apache.pulsar.common.policies.data.BookieAffinityGroupData;
import org.apache.pulsar.common.policies.data.BundlesData;
+import org.apache.pulsar.common.policies.data.ClusterData;
import org.apache.pulsar.common.policies.data.DispatchRate;
import org.apache.pulsar.common.policies.data.EntryFilters;
import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode;
@@ -71,16 +83,19 @@ import org.apache.pulsar.common.policies.data.PolicyOperation;
import org.apache.pulsar.common.policies.data.PublishRate;
import org.apache.pulsar.common.policies.data.RetentionPolicies;
import org.apache.pulsar.common.policies.data.SubscribeRate;
+import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
import org.apache.pulsar.common.policies.data.TenantInfo;
import org.apache.pulsar.packages.management.core.MockedPackagesStorageProvider;
import org.apache.pulsar.packages.management.core.common.PackageMetadata;
import org.apache.pulsar.security.MockedPulsarStandalone;
+import org.awaitility.Awaitility;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test(groups = "broker-admin")
@@ -100,6 +115,8 @@ public class NamespaceAuthZTest extends MockedPulsarStandalone {
private volatile Consumer<InvocationOnMock> allowNamespacePolicyOperationAsyncHandler;
private volatile Consumer<InvocationOnMock> allowNamespaceOperationAsyncHandler;
+ // when set, replaces the result of tenant operation checks
+ private volatile Supplier<CompletableFuture<Boolean>> tenantOperationResult;
@SneakyThrows
@BeforeClass
@@ -108,6 +125,7 @@ public class NamespaceAuthZTest extends MockedPulsarStandalone {
getServiceConfiguration().setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName());
getServiceConfiguration().setDefaultNumberOfNamespaceBundles(1);
getServiceConfiguration().setForceDeleteNamespaceAllowed(true);
+ getServiceConfiguration().setEnableShadowTopics(true);
configureTokenAuthentication();
configureDefaultAuthorization();
start();
@@ -145,6 +163,13 @@ public class NamespaceAuthZTest extends MockedPulsarStandalone {
return invocationOnMock.callRealMethod();
}).when(authorizationService).allowNamespaceOperationAsync(Mockito.any(), Mockito.any(), Mockito.any(),
Mockito.any());
+ Mockito.doAnswer(invocationOnMock -> {
+ Supplier<CompletableFuture<Boolean>> localTenantOperationResult = tenantOperationResult;
+ return localTenantOperationResult != null
+ ? localTenantOperationResult.get() : invocationOnMock.callRealMethod();
+ })
+ .when(authorizationService).allowTenantOperationAsync(Mockito.any(), Mockito.any(),
+ Mockito.any(), Mockito.any());
}
@@ -166,6 +191,7 @@ public class NamespaceAuthZTest extends MockedPulsarStandalone {
@AfterMethod
public void after() throws Exception {
+ tenantOperationResult = null;
deleteNamespaceWithRetry("public/default", true, superUserAdmin);
superUserAdmin.namespaces().createNamespace("public/default");
allowNamespacePolicyOperationAsyncHandler = null;
@@ -908,6 +934,860 @@ public class NamespaceAuthZTest extends MockedPulsarStandalone {
superUserAdmin.topics().delete(topic, true);
}
+ @Test
+ public void testNamespaceSubscriptionOperationsApplySubscriptionPolicies() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String otherSub = "other-sub";
+ final String ownSub = subject + "-sub";
+ final String ownSub2 = subject + "-sub2";
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ for (String sub : List.of(otherSub, ownSub, ownSub2)) {
+ superUserAdmin.topics().createSubscription(topic, sub, MessageId.earliest);
+ }
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes());
+ }
+
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+
+ // subscriptions that do not match the role prefix are rejected
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().unsubscribeNamespace(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, otherSub));
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+
+ // subscriptions that match the role prefix are allowed on the namespace and on the bundle
+ subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, ownSub);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+ subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle, ownSub2);
+ assertEquals(getMsgBacklog(topic, ownSub2), 0);
+ subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, ownSub2);
+ assertNull(superUserAdmin.topics().getStats(topic).getSubscriptions().get(ownSub2));
+ subAdmin.namespaces().unsubscribeNamespace(namespace, ownSub);
+ assertNull(superUserAdmin.topics().getStats(topic).getSubscriptions().get(ownSub));
+
+ // subscription roles are applied
+ superUserAdmin.topics().createSubscription(topic, ownSub, MessageId.earliest);
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.None);
+ superUserAdmin.namespaces().grantPermissionOnSubscription(namespace, otherSub, Set.of("other-role"));
+ superUserAdmin.namespaces().grantPermissionOnSubscription(namespace, ownSub, Set.of(subject));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().unsubscribeNamespace(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, otherSub));
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+
+ // roles listed for the subscription are allowed
+ subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle, ownSub);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+ subAdmin.namespaces().unsubscribeNamespaceBundle(namespace, defaultBundle, ownSub);
+ assertNull(superUserAdmin.topics().getStats(topic).getSubscriptions().get(ownSub));
+
+ // replicator cursor names require tenant admin permission
+ final String replicatorCursor = getPulsarService().getConfiguration().getReplicatorPrefix() + ".remote";
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, replicatorCursor));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ replicatorCursor));
+
+ // super users and tenant admins are not affected by subscription policies
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ superUserAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle, otherSub);
+ assertEquals(getMsgBacklog(topic, otherSub), 0);
+ tenantManagerAdmin.namespaces().unsubscribeNamespace(namespace, otherSub);
+ assertNull(superUserAdmin.topics().getStats(topic).getSubscriptions().get(otherSub));
+
+ producer.close();
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testNamespaceClearBacklogForSubscriptionWithReplicator() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String remoteCluster = "remote-" + random;
+ final String replicatorCursor =
+ getPulsarService().getConfiguration().getReplicatorPrefix() + "." + remoteCluster;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+
+ // the remote cluster is not reachable, so the replicator keeps its backlog
+ superUserAdmin.clusters().createCluster(remoteCluster, ClusterData.builder()
+ .serviceUrl("http://127.0.0.1:1")
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster, remoteCluster))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ // an ordinary subscription with the same name as the remote cluster
+ superUserAdmin.topics().createSubscription(topic, remoteCluster, MessageId.earliest);
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster, remoteCluster),
+ false);
+ Awaitility.await().untilAsserted(() -> assertNotNull(
+ superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster)));
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ assertEquals(getMsgBacklog(topic, remoteCluster), numMessages);
+
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ // the prefixed replicator cursor name is rejected for ordinary roles
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, replicatorCursor));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ replicatorCursor));
+ // on the namespace, a plain name clears the subscription with that name for ordinary roles
+ subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster);
+ assertEquals(getMsgBacklog(topic, remoteCluster), 0);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+
+ // on the bundle, a plain name clears the subscription with that name for ordinary roles
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ 2 * numMessages));
+ assertEquals(getMsgBacklog(topic, remoteCluster), numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle, remoteCluster);
+ assertEquals(getMsgBacklog(topic, remoteCluster), 0);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), 2 * numMessages);
+
+ // without such a subscription, a plain name that resolves to the replicator is rejected for ordinary roles
+ superUserAdmin.topics().deleteSubscription(topic, remoteCluster);
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ remoteCluster));
+ assertEquals(getReplicationBacklog(topic, remoteCluster), 2 * numMessages);
+
+ // tenant admins keep the existing behaviour: a plain name clears the subscription with that name first
+ superUserAdmin.topics().createSubscription(topic, remoteCluster, MessageId.earliest);
+ assertEquals(getMsgBacklog(topic, remoteCluster), 2 * numMessages);
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ remoteCluster);
+ assertEquals(getMsgBacklog(topic, remoteCluster), 0);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), 2 * numMessages);
+
+ // without such a subscription, a plain cluster name clears the replicator backlog on the bundle ...
+ superUserAdmin.topics().deleteSubscription(topic, remoteCluster);
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ remoteCluster);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+
+ // ... and on the namespace
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ tenantManagerAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+
+ // the prefixed replicator cursor name also works for tenant admins
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ tenantManagerAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, replicatorCursor);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+
+ producer.close();
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster), false);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ superUserAdmin.tenants().deleteTenant(tenant);
+ superUserAdmin.clusters().deleteCluster(remoteCluster);
+ }
+
+ @Test
+ public void testNamespaceClearBacklogForSubscriptionWithTopicLevelReplication() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String remoteCluster = "remote-" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+
+ // the remote cluster is not reachable, so the replicator keeps its backlog
+ superUserAdmin.clusters().createCluster(remoteCluster, ClusterData.builder()
+ .serviceUrl("http://127.0.0.1:1")
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster, remoteCluster))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topicPolicies().setReplicationClusters(topic, List.of(localCluster, remoteCluster));
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ Awaitility.await().untilAsserted(() -> assertNotNull(
+ superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster)));
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+
+ // the remote cluster is no longer allowed for the tenant, the topic level replication is still configured
+ superUserAdmin.tenants().updateTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster))
+ .build());
+ assertNotNull(superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster));
+
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ // the plain cluster name resolves to the replicator and is rejected for ordinary roles
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ remoteCluster));
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+
+ // tenant admins keep the existing behaviour
+ tenantManagerAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+
+ producer.close();
+ superUserAdmin.topicPolicies().removeReplicationClusters(topic);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ superUserAdmin.tenants().deleteTenant(tenant);
+ superUserAdmin.clusters().deleteCluster(remoteCluster);
+ }
+
+ @Test
+ public void testNamespaceClearBacklogForSubscriptionWithShadowTopic() throws Exception {
+ verifyNamespaceClearBacklogForSubscriptionWithShadowTopic(false);
+ }
+
+ @Test
+ public void testNamespaceClearBacklogForSubscriptionWithShortShadowTopicName() throws Exception {
+ verifyNamespaceClearBacklogForSubscriptionWithShadowTopic(true);
+ }
+
+ private void verifyNamespaceClearBacklogForSubscriptionWithShadowTopic(boolean shortName) throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String shadowNamespace = "public/" + random + "-shadow";
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String shadowTopicName = shadowNamespace + "/" + random;
+ // shadow topics can be configured with the full or the short topic name, the configured name is the key of
+ // the shadow replicator
+ final String shadowTopic = shortName ? shadowTopicName : "persistent://" + shadowTopicName;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String defaultBundle = "0x00000000_0xffffffff";
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.namespaces().createNamespace(shadowNamespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createShadowTopic("persistent://" + shadowTopicName, topic);
+ superUserAdmin.topics().setShadowTopics(topic, List.of(shadowTopic));
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ sendMessages(producer, 1);
+ PersistentTopic persistentTopic = (PersistentTopic) getPulsarService().getBrokerService()
+ .getTopicIfExists(topic).get().orElseThrow();
+ Awaitility.await().untilAsserted(() ->
+ assertTrue(persistentTopic.getShadowReplicators().containsKey(shadowTopic)));
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ // the shadow replicator cursor is rejected for ordinary roles
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, shadowTopic));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ shadowTopic));
+
+ // tenant admins keep the existing behaviour
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle,
+ shadowTopic);
+ tenantManagerAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, shadowTopic);
+
+ producer.close();
+ superUserAdmin.topics().removeShadowTopics(topic);
+ deleteNamespaceWithRetry(shadowNamespace, true, superUserAdmin);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testNamespaceClearBacklogForSubscriptionNamedAfterCluster() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ // a remote cluster which is allowed for the tenant, replication is not configured
+ final String otherCluster = "other-" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+ superUserAdmin.clusters().createCluster(otherCluster, ClusterData.builder()
+ .serviceUrl("http://127.0.0.1:1")
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(getPulsarService().getConfiguration().getClusterName(), otherCluster))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, otherCluster, MessageId.earliest);
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ sendMessages(producer, numMessages);
+ assertEquals(getMsgBacklog(topic, otherCluster), numMessages);
+ subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, otherCluster);
+ assertEquals(getMsgBacklog(topic, otherCluster), 0);
+
+ sendMessages(producer, numMessages);
+ assertEquals(getMsgBacklog(topic, otherCluster), numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, defaultBundle, otherCluster);
+ assertEquals(getMsgBacklog(topic, otherCluster), 0);
+
+ producer.close();
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ superUserAdmin.tenants().deleteTenant(tenant);
+ superUserAdmin.clusters().deleteCluster(otherCluster);
+ }
+
+ @Test
+ public void testClearNamespaceBacklogAppliesSubscriptionPolicies() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String partitionedTopic = "persistent://" + namespace + "/" + random + "-partitioned";
+ final List<String> topics = List.of("persistent://" + namespace + "/" + random + "-1",
+ "persistent://" + namespace + "/" + random + "-2",
+ partitionedTopic + "-partition-0", partitionedTopic + "-partition-1");
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String otherSub = "other-sub";
+ final String ownSub = subject + "-sub";
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topics.get(0));
+ superUserAdmin.topics().createNonPartitionedTopic(topics.get(1));
+ superUserAdmin.topics().createPartitionedTopic(partitionedTopic, 2);
+ final List<Producer<byte[]>> producers = new ArrayList<>();
+ for (String topic : topics) {
+ superUserAdmin.topics().createSubscription(topic, otherSub, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(topic, ownSub, MessageId.earliest);
+ producers.add(pulsarClient.newProducer().topic(topic).enableBatching(false).create());
+ }
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+
+ // only the subscriptions that match the role prefix are cleared, on the namespace and on the bundle
+ sendMessages(producers, numMessages);
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertBacklogs(topics, ownSub, 0, otherSub, numMessages);
+ sendMessages(producers, numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertBacklogs(topics, ownSub, 0, otherSub, 2 * numMessages);
+
+ // subscription roles are applied
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.None);
+ superUserAdmin.namespaces().grantPermissionOnSubscription(namespace, otherSub, Set.of("other-role"));
+ sendMessages(producers, numMessages);
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertBacklogs(topics, ownSub, 0, otherSub, 3 * numMessages);
+ sendMessages(producers, numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertBacklogs(topics, ownSub, 0, otherSub, 4 * numMessages);
+
+ // without subscription policies, all subscriptions are cleared
+ superUserAdmin.namespaces().revokePermissionOnSubscription(namespace, otherSub, "other-role");
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertBacklogs(topics, ownSub, 0, otherSub, 0);
+ sendMessages(producers, numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertBacklogs(topics, ownSub, 0, otherSub, 0);
+
+ // tenant admins are not affected by subscription policies
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ sendMessages(producers, numMessages);
+ tenantManagerAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertBacklogs(topics, ownSub, 0, otherSub, 0);
+ sendMessages(producers, numMessages);
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertBacklogs(topics, ownSub, 0, otherSub, 0);
+
+ for (Producer<byte[]> producer : producers) {
+ producer.close();
+ }
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testClearNamespaceBacklogWithReplicator() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String remoteCluster = "remote-" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String sub = "sub";
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+
+ // the remote cluster is not reachable, so the replicator keeps its backlog
+ superUserAdmin.clusters().createCluster(remoteCluster, ClusterData.builder()
+ .serviceUrl("http://127.0.0.1:1")
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster, remoteCluster))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, sub, MessageId.earliest);
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster, remoteCluster),
+ false);
+ Awaitility.await().untilAsserted(() -> assertNotNull(
+ superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster)));
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ // ordinary roles clear the subscriptions but not the replicator, on the namespace and on the bundle
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertEquals(getMsgBacklog(topic, sub), 0);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ 2 * numMessages));
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertEquals(getMsgBacklog(topic, sub), 0);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), 2 * numMessages);
+
+ // tenant admins keep clearing the replicator backlog
+ tenantManagerAdmin.namespaces().clearNamespaceBacklog(namespace);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+ assertEquals(getMsgBacklog(topic, sub), 0);
+
+ producer.close();
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster), false);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testClearNamespaceBacklogKeepsShadowReplicator() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String shadowNamespace = "public/" + random + "-shadow";
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String shadowTopic = "persistent://" + shadowNamespace + "/" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String sub = "sub";
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.namespaces().createNamespace(shadowNamespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, sub, MessageId.earliest);
+ superUserAdmin.topics().createShadowTopic(shadowTopic, topic);
+ superUserAdmin.topics().setShadowTopics(topic, List.of(shadowTopic));
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ sendMessages(producer, 1);
+ PersistentTopic persistentTopic = (PersistentTopic) getPulsarService().getBrokerService()
+ .getTopicIfExists(topic).get().orElseThrow();
+ Awaitility.await().untilAsserted(() ->
+ assertTrue(persistentTopic.getShadowReplicators().containsKey(shadowTopic)));
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+ // records the clear backlog calls on the shadow replicator
+ final PersistentReplicator shadowReplicator =
+ (PersistentReplicator) persistentTopic.getShadowReplicators().get(shadowTopic);
+ final PersistentReplicator shadowReplicatorSpy = Mockito.spy(shadowReplicator);
+ persistentTopic.getShadowReplicators().put(shadowTopic, shadowReplicatorSpy);
+ try {
+ // ordinary roles clear the subscriptions but not the shadow replicator
+ sendMessages(producer, numMessages);
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertEquals(getMsgBacklog(topic, sub), 0);
+ sendMessages(producer, numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertEquals(getMsgBacklog(topic, sub), 0);
+ Mockito.verify(shadowReplicatorSpy, Mockito.never()).clearBacklog();
+
+ // tenant admins keep clearing the shadow replicator
+ tenantManagerAdmin.namespaces().clearNamespaceBacklog(namespace);
+ Mockito.verify(shadowReplicatorSpy, Mockito.times(1)).clearBacklog();
+ tenantManagerAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ Mockito.verify(shadowReplicatorSpy, Mockito.times(2)).clearBacklog();
+ } finally {
+ persistentTopic.getShadowReplicators().replace(shadowTopic, shadowReplicatorSpy, shadowReplicator);
+ }
+
+ producer.close();
+ superUserAdmin.topics().removeShadowTopics(topic);
+ deleteNamespaceWithRetry(shadowNamespace, true, superUserAdmin);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testClearNamespaceBacklogRedirectsToPeerCluster() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String peerCluster = "peer-" + random;
+ final String peerServiceUrl = "http://127.0.0.1:1";
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ superUserAdmin.clusters().createCluster(peerCluster, ClusterData.builder()
+ .serviceUrl(peerServiceUrl)
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.clusters().updatePeerClusterNames(localCluster, new LinkedHashSet<>(List.of(peerCluster)));
+ try {
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster, peerCluster))
+ .build());
+ // the namespace is served by the peer cluster, so this cluster has no topics of it
+ superUserAdmin.namespaces().createNamespace(namespace, Set.of(peerCluster));
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+
+ // the request is redirected to the peer cluster instead of clearing the local topics only
+ HttpRequest request = HttpRequest.newBuilder(URI.create(getPulsarService().getWebServiceAddress()
+ + "/admin/v2/namespaces/" + namespace + "/clearBacklog"))
+ .header("Authorization", "Bearer " + token)
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.noBody())
+ .build();
+ HttpResponse<Void> response = HttpClient.newHttpClient().send(request,
+ HttpResponse.BodyHandlers.discarding());
+ assertEquals(response.statusCode(), 307);
+ assertTrue(response.headers().firstValue("Location").orElseThrow().startsWith(peerServiceUrl));
+
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster), false);
+ } finally {
+ superUserAdmin.clusters().updatePeerClusterNames(localCluster, null);
+ }
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ superUserAdmin.tenants().deleteTenant(tenant);
+ superUserAdmin.clusters().deleteCluster(peerCluster);
+ }
+
+ @DataProvider
+ public Object[][] tenantOperationResults() {
+ return new Object[][]{{"unsupported"}, {"denied"}};
+ }
+
+ @Test(dataProvider = "tenantOperationResults")
+ public void testClearNamespaceBacklogDoesNotDependOnTenantOperations(String tenantOperationMode)
+ throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String remoteCluster = "remote-" + random;
+ final int numMessages = 5;
+ createReplicatedNamespace(tenant, namespace, topic, localCluster, remoteCluster);
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ tenantOperationResult = "unsupported".equals(tenantOperationMode)
+ ? () -> CompletableFuture.failedFuture(new IllegalStateException("tenant operations not supported"))
+ : () -> CompletableFuture.completedFuture(false);
+
+ // super users and tenant admins are recognized with the provider's admin checks
+ for (PulsarAdmin admin : List.of(superUserAdmin, tenantManagerAdmin)) {
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ admin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ admin.namespaces().clearNamespaceBacklog(namespace);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster), 0));
+ }
+
+ tenantOperationResult = null;
+ producer.close();
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster), false);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ @Test
+ public void testClearNamespaceBacklogIgnoresTenantOperationsGrantedToConsumers() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String localCluster = getPulsarService().getConfiguration().getClusterName();
+ final String remoteCluster = "remote-" + random;
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String otherSub = "other-sub";
+ final String ownSub = subject + "-sub";
+ final String defaultBundle = "0x00000000_0xffffffff";
+ final int numMessages = 5;
+ createReplicatedNamespace(tenant, namespace, topic, localCluster, remoteCluster);
+ superUserAdmin.topics().createSubscription(topic, otherSub, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(topic, ownSub, MessageId.earliest);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ @Cleanup
+ Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ // the provider grants tenant operations to every role
+ tenantOperationResult = () -> CompletableFuture.completedFuture(true);
+
+ sendMessages(producer, numMessages);
+ Awaitility.await().untilAsserted(() -> assertEquals(getReplicationBacklog(topic, remoteCluster),
+ numMessages));
+ subAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+ subAdmin.namespaces().clearNamespaceBundleBacklog(namespace, defaultBundle);
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+ try {
+ subAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, remoteCluster);
+ } catch (PulsarAdminException e) {
+ // rejected or ignored, the replicator keeps its backlog either way
+ }
+ assertEquals(getReplicationBacklog(topic, remoteCluster), numMessages);
+
+ tenantOperationResult = null;
+ producer.close();
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster), false);
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ private void createReplicatedNamespace(String tenant, String namespace, String topic, String localCluster,
+ String remoteCluster) throws Exception {
+ // the remote cluster is not reachable, so the replicator keeps its backlog
+ superUserAdmin.clusters().createCluster(remoteCluster, ClusterData.builder()
+ .serviceUrl("http://127.0.0.1:1")
+ .brokerServiceUrl("pulsar://127.0.0.1:1")
+ .build());
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(TENANT_ADMIN_SUBJECT))
+ .allowedClusters(Set.of(localCluster, remoteCluster))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.namespaces().setNamespaceReplicationClusters(namespace, Set.of(localCluster, remoteCluster),
+ false);
+ Awaitility.await().untilAsserted(() -> assertNotNull(
+ superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster)));
+ }
+
+ @Test
+ public void testExpireMessagesForAllSubscriptionsAppliesSubscriptionPolicies() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String prefix = "persistent://" + namespace + "/" + random;
+ // each subscription is expired only once, so that no expiry is still running when it is checked
+ final String topic = prefix + "-topic";
+ final String partitionedTopic = prefix + "-partitioned";
+ final String partitionedTopic2 = prefix + "-partitioned-2";
+ final String adminTopic = prefix + "-admin";
+ final String adminPartitionedTopic = prefix + "-admin-partitioned";
+ final String subject = UUID.randomUUID().toString();
+ final String token = Jwts.builder()
+ .claim("sub", subject).signWith(SECRET_KEY).compact();
+ final String otherSub = "other-sub";
+ final String ownSub = subject + "-sub";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ final List<String> allTopics = new ArrayList<>();
+ for (String t : List.of(topic, adminTopic)) {
+ superUserAdmin.topics().createNonPartitionedTopic(t);
+ allTopics.add(t);
+ }
+ for (String t : List.of(partitionedTopic, partitionedTopic2, adminPartitionedTopic)) {
+ superUserAdmin.topics().createPartitionedTopic(t, 2);
+ allTopics.add(t + "-partition-0");
+ allTopics.add(t + "-partition-1");
+ }
+ final List<Producer<byte[]>> producers = new ArrayList<>();
+ for (String t : allTopics) {
+ superUserAdmin.topics().createSubscription(t, otherSub, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(t, ownSub, MessageId.earliest);
+ producers.add(pulsarClient.newProducer().topic(t).enableBatching(false).create());
+ }
+ @Cleanup final PulsarAdmin subAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(token))
+ .build();
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, subject, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ sendMessages(producers, numMessages);
+ // the messages must be older than the expiry time
+ Thread.sleep(1500);
+
+ // only the subscriptions that match the role prefix are expired, on partitioned topics too
+ subAdmin.topics().expireMessagesForAllSubscriptions(topic, 1);
+ subAdmin.topics().expireMessagesForAllSubscriptions(partitionedTopic, 1);
+ final List<String> expired = List.of(topic, partitionedTopic + "-partition-0",
+ partitionedTopic + "-partition-1");
+ Awaitility.await().untilAsserted(() -> assertBacklogs(expired, ownSub, 0, otherSub, numMessages));
+
+ // the same applies to a partition
+ subAdmin.topics().expireMessagesForAllSubscriptions(partitionedTopic2 + "-partition-0", 1);
+ Awaitility.await().untilAsserted(() -> assertBacklogs(List.of(partitionedTopic2 + "-partition-0"),
+ ownSub, 0, otherSub, numMessages));
+ assertBacklogs(List.of(partitionedTopic2 + "-partition-1"), ownSub, numMessages, otherSub, numMessages);
+
+ // tenant admins are not affected by subscription policies
+ tenantManagerAdmin.topics().expireMessagesForAllSubscriptions(adminTopic, 1);
+ tenantManagerAdmin.topics().expireMessagesForAllSubscriptions(adminPartitionedTopic, 1);
+ Awaitility.await().untilAsserted(() -> assertBacklogs(List.of(adminTopic,
+ adminPartitionedTopic + "-partition-0", adminPartitionedTopic + "-partition-1"),
+ ownSub, 0, otherSub, 0));
+
+ for (Producer<byte[]> producer : producers) {
+ producer.close();
+ }
+ deleteNamespaceWithRetry(namespace, true, superUserAdmin);
+ }
+
+ private void assertBacklogs(List<String> topics, String sub1, long backlog1, String sub2, long backlog2)
+ throws PulsarAdminException {
+ for (String topic : topics) {
+ assertEquals(getMsgBacklog(topic, sub1), backlog1, topic + " " + sub1);
+ assertEquals(getMsgBacklog(topic, sub2), backlog2, topic + " " + sub2);
+ }
+ }
+
+ private static void sendMessages(List<Producer<byte[]>> producers, int numMessages) throws Exception {
+ for (Producer<byte[]> producer : producers) {
+ sendMessages(producer, numMessages);
+ }
+ }
+
+ private static void sendMessages(Producer<byte[]> producer, int numMessages) throws Exception {
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes());
+ }
+ }
+
+ private long getMsgBacklog(String topic, String subscription) throws PulsarAdminException {
+ return superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog();
+ }
+
+ private long getReplicationBacklog(String topic, String remoteCluster) throws PulsarAdminException {
+ return superUserAdmin.topics().getStats(topic).getReplication().get(remoteCluster).getReplicationBacklog();
+ }
+
@Test
public void testPackageAPI() throws Exception {
final String namespace = "public/default";
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionAnonymousRoleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionAnonymousRoleTest.java
new file mode 100644
index 000000000000..20d4c1dce9f0
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionAnonymousRoleTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertEquals;
+import java.util.Set;
+import java.util.UUID;
+import lombok.Cleanup;
+import lombok.SneakyThrows;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
+import org.apache.pulsar.security.MockedPulsarStandalone;
+import org.awaitility.Awaitility;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Namespace and topic subscription operations performed by the configured anonymous role.
+ */
+@Test(groups = "broker-admin")
+public class NamespaceSubscriptionAnonymousRoleTest extends MockedPulsarStandalone {
+
+ private static final String ANONYMOUS_ROLE = "anonymous-role";
+ private static final String DEFAULT_BUNDLE = "0x00000000_0xffffffff";
+
+ private PulsarAdmin superUserAdmin;
+ private PulsarAdmin anonymousAdmin;
+
+ @SneakyThrows
+ @BeforeClass
+ public void setup() {
+ getServiceConfiguration().setDefaultNumberOfNamespaceBundles(1);
+ getServiceConfiguration().setForceDeleteNamespaceAllowed(true);
+ configureTokenAuthentication();
+ configureDefaultAuthorization();
+ getServiceConfiguration().setAnonymousUserRole(ANONYMOUS_ROLE);
+ start();
+ superUserAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(SUPER_USER_TOKEN))
+ .build();
+ anonymousAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .build();
+ }
+
+ @SneakyThrows
+ @AfterClass(alwaysRun = true)
+ public void cleanup() {
+ if (anonymousAdmin != null) {
+ anonymousAdmin.close();
+ anonymousAdmin = null;
+ }
+ if (superUserAdmin != null) {
+ superUserAdmin.close();
+ superUserAdmin = null;
+ }
+ close();
+ }
+
+ @Test
+ public void testNamespaceSubscriptionOperationsApplySubscriptionPolicies() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String otherSub = "other-sub";
+ final String ownSub = ANONYMOUS_ROLE + "-sub";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, otherSub, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(topic, ownSub, MessageId.earliest);
+ @Cleanup
+ Producer<byte[]> producer = getPulsarService().getClient().newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes());
+ }
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, ANONYMOUS_ROLE, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace,
+ DEFAULT_BUNDLE, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.namespaces().unsubscribeNamespace(namespace, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.namespaces().unsubscribeNamespaceBundle(namespace, DEFAULT_BUNDLE, otherSub));
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+
+ anonymousAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, ownSub);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+
+ // clearing the backlog of all subscriptions only clears the subscriptions the role may access
+ sendMessages(producer, numMessages);
+ anonymousAdmin.namespaces().clearNamespaceBacklog(namespace);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+ assertEquals(getMsgBacklog(topic, otherSub), 2 * numMessages);
+
+ producer.close();
+ superUserAdmin.namespaces().deleteNamespace(namespace, true);
+ }
+
+ @Test
+ public void testTopicSubscriptionOperationsApplySubscriptionPolicies() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String namespace = "public/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String otherSub = "other-sub";
+ final String ownSub = ANONYMOUS_ROLE + "-sub";
+ final int numMessages = 5;
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, otherSub, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(topic, ownSub, MessageId.earliest);
+ @Cleanup
+ Producer<byte[]> producer = getPulsarService().getClient().newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ sendMessages(producer, numMessages);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, ANONYMOUS_ROLE, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+
+ // the topic operations reject subscriptions outside the role prefix with a server error
+ Assert.assertThrows(PulsarAdminException.class,
+ () -> anonymousAdmin.topics().skipAllMessages(topic, otherSub));
+ Assert.assertThrows(PulsarAdminException.class,
+ () -> anonymousAdmin.topics().resetCursor(topic, otherSub, MessageId.latest));
+ Assert.assertThrows(PulsarAdminException.class,
+ () -> anonymousAdmin.topics().deleteSubscription(topic, otherSub));
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+
+ anonymousAdmin.topics().skipAllMessages(topic, ownSub);
+ assertEquals(getMsgBacklog(topic, ownSub), 0);
+
+ // subscription roles are applied
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.None);
+ superUserAdmin.namespaces().grantPermissionOnSubscription(namespace, otherSub, Set.of("other-role"));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.topics().skipAllMessages(topic, otherSub));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.topics().skipMessages(topic, otherSub, 1));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.topics().resetCursor(topic, otherSub, MessageId.latest));
+ Assert.assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> anonymousAdmin.topics().deleteSubscription(topic, otherSub));
+ assertEquals(getMsgBacklog(topic, otherSub), numMessages);
+
+ // expiring messages of all subscriptions only expires the subscriptions the role may access
+ sendMessages(producer, numMessages);
+ Thread.sleep(1500);
+ anonymousAdmin.topics().expireMessagesForAllSubscriptions(topic, 1);
+ Awaitility.await().untilAsserted(() -> assertEquals(getMsgBacklog(topic, ownSub), 0));
+ assertEquals(getMsgBacklog(topic, otherSub), 2 * numMessages);
+
+ producer.close();
+ superUserAdmin.namespaces().deleteNamespace(namespace, true);
+ }
+
+ private static void sendMessages(Producer<byte[]> producer, int numMessages) throws Exception {
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes());
+ }
+ }
+
+ private long getMsgBacklog(String topic, String subscription) throws PulsarAdminException {
+ return superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionProxyRoleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionProxyRoleTest.java
new file mode 100644
index 000000000000..2a4ada995dc6
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespaceSubscriptionProxyRoleTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.admin;
+
+import static org.testng.Assert.assertEquals;
+import io.jsonwebtoken.Jwts;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import lombok.Cleanup;
+import lombok.SneakyThrows;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.apache.pulsar.security.MockedPulsarStandalone;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Namespace clear backlog operations for replicator cursor names on requests forwarded by a proxy.
+ */
+@Test(groups = "broker-admin")
+public class NamespaceSubscriptionProxyRoleTest extends MockedPulsarStandalone {
+
+ private static final String DEFAULT_BUNDLE = "0x00000000_0xffffffff";
+ private static final String ADMIN_PROXY_ROLE = "tenant-admin-proxy";
+ private static final String SUPER_USER_PROXY_ROLE = "super-user-proxy";
+ private static final String TENANT_ADMIN_ROLE = "tenant-admin";
+ private static final String CONSUMER_ROLE = "consumer";
+
+ private PulsarAdmin superUserAdmin;
+ private HttpClient httpClient;
+
+ @SneakyThrows
+ @BeforeClass
+ public void setup() {
+ getServiceConfiguration().setDefaultNumberOfNamespaceBundles(1);
+ getServiceConfiguration().setForceDeleteNamespaceAllowed(true);
+ configureTokenAuthentication();
+ configureDefaultAuthorization();
+ Set<String> superUserRoles = new HashSet<>(getServiceConfiguration().getSuperUserRoles());
+ superUserRoles.add(SUPER_USER_PROXY_ROLE);
+ getServiceConfiguration().setSuperUserRoles(superUserRoles);
+ getServiceConfiguration().setProxyRoles(Set.of(ADMIN_PROXY_ROLE, SUPER_USER_PROXY_ROLE));
+ start();
+ superUserAdmin = PulsarAdmin.builder()
+ .serviceHttpUrl(getPulsarService().getWebServiceAddress())
+ .authentication(new AuthenticationToken(SUPER_USER_TOKEN))
+ .build();
+ httpClient = HttpClient.newHttpClient();
+ }
+
+ @SneakyThrows
+ @AfterClass(alwaysRun = true)
+ public void cleanup() {
+ httpClient = null;
+ if (superUserAdmin != null) {
+ superUserAdmin.close();
+ superUserAdmin = null;
+ }
+ close();
+ }
+
+ @Test
+ public void testReplicatorCursorNameThroughProxy() throws Exception {
+ final String random = UUID.randomUUID().toString();
+ final String tenant = "tenant-" + random;
+ final String namespace = tenant + "/" + random;
+ final String topic = "persistent://" + namespace + "/" + random;
+ final String subscription = "sub-" + random;
+ // resolved to the subscription with the same name as the remote cluster part of the name
+ final String replicatorCursor =
+ getPulsarService().getConfiguration().getReplicatorPrefix() + "." + subscription;
+ final int numMessages = 5;
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(ADMIN_PROXY_ROLE, TENANT_ADMIN_ROLE))
+ .allowedClusters(Set.of(getPulsarService().getConfiguration().getClusterName()))
+ .build());
+ superUserAdmin.namespaces().createNamespace(namespace, 1);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, CONSUMER_ROLE, Set.of(AuthAction.consume));
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, subscription, MessageId.earliest);
+ @Cleanup
+ Producer<byte[]> producer = getPulsarService().getClient().newProducer().topic(topic)
+ .enableBatching(false)
+ .create();
+ sendMessages(producer, numMessages);
+
+ // a tenant admin proxy forwarding a consumer: the consumer is not allowed to use replicator cursor names
+ assertEquals(clearBacklog(ADMIN_PROXY_ROLE, CONSUMER_ROLE, namespace, null, replicatorCursor), 403);
+ assertEquals(clearBacklog(ADMIN_PROXY_ROLE, CONSUMER_ROLE, namespace, DEFAULT_BUNDLE, replicatorCursor),
+ 403);
+ assertEquals(getMsgBacklog(topic, subscription), numMessages);
+
+ // a super user proxy forwarding a tenant admin keeps the existing behaviour
+ assertEquals(clearBacklog(SUPER_USER_PROXY_ROLE, TENANT_ADMIN_ROLE, namespace, null, replicatorCursor),
+ 204);
+ assertEquals(getMsgBacklog(topic, subscription), 0);
+ sendMessages(producer, numMessages);
+ assertEquals(clearBacklog(SUPER_USER_PROXY_ROLE, TENANT_ADMIN_ROLE, namespace, DEFAULT_BUNDLE,
+ replicatorCursor), 204);
+ assertEquals(getMsgBacklog(topic, subscription), 0);
+
+ producer.close();
+ superUserAdmin.namespaces().deleteNamespace(namespace, true);
+ superUserAdmin.tenants().deleteTenant(tenant);
+ }
+
+ private int clearBacklog(String proxyRole, String originalPrincipal, String namespace, String bundle,
+ String subscription) throws Exception {
+ final String proxyToken = Jwts.builder().claim("sub", proxyRole).signWith(SECRET_KEY).compact();
+ final String path = "/admin/v2/namespaces/" + namespace + (bundle != null ? "/" + bundle : "")
+ + "/clearBacklog/" + subscription;
+ HttpRequest request = HttpRequest.newBuilder(URI.create(getPulsarService().getWebServiceAddress() + path))
+ .header("Authorization", "Bearer " + proxyToken)
+ .header("X-Original-Principal", originalPrincipal)
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.noBody())
+ .build();
+ return httpClient.send(request, HttpResponse.BodyHandlers.discarding()).statusCode();
+ }
+
+ private static void sendMessages(Producer<byte[]> producer, int numMessages) throws Exception {
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes());
+ }
+ }
+
+ private long getMsgBacklog(String topic, String subscription) throws PulsarAdminException {
+ return superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java
index 16d53b6f96a8..4c1ba5bf47e5 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java
@@ -2604,7 +2604,7 @@ public class NamespacesTest extends MockedPulsarServiceBaseTest {
// Test clear backlog for subscription on non-existent namespace - should return 404
AsyncResponse clearSubResponse = mock(AsyncResponse.class);
namespaces.clearNamespaceBundleBacklogForSubscription(clearSubResponse, testTenant, nonExistentNs,
- bundleRange, "test-sub", false);
+ bundleRange, "test-sub", false, false);
ArgumentCaptor<RestException> clearSubCaptor = ArgumentCaptor.forClass(RestException.class);
verify(clearSubResponse, timeout(5000).times(1)).resume(clearSubCaptor.capture());
assertEquals(clearSubCaptor.getValue().getResponse().getStatus(),
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
index 2deebd32a38a..e91454c3afd7 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiRolesTokenAuthorizationProviderTest.java
@@ -48,9 +48,12 @@ import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.impl.auth.AuthenticationToken;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.AuthAction;
import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.SubscriptionAuthMode;
import org.apache.pulsar.common.policies.data.TenantInfo;
+import org.awaitility.Awaitility;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -239,6 +242,228 @@ public class MultiRolesTokenAuthorizationProviderTest extends MockedPulsarServic
});
}
+ @Test
+ public void testNamespaceClearBacklogWithSecondaryTenantAdminRole() throws Exception {
+ final String tenant = "multi-roles-clear-backlog-tenant";
+ final String namespace = tenant + "/ns";
+ final String topic = "persistent://" + namespace + "/test-topic";
+ final String subscription = "sub";
+ final String bundle = "0x00000000_0xffffffff";
+ // resolved to the subscription with the same name as the remote cluster part of the name
+ final String replicatorCursor = pulsar.getConfiguration().getReplicatorPrefix() + "." + subscription;
+ final int numMessages = 5;
+ Map<String, Object> claims = new HashMap<>();
+ // the first role is the primary role of the token, the second one is a tenant admin role
+ claims.put("roles", List.of("clear-backlog-consumer", "clear-backlog-tenant-admin"));
+ final String tenantAdminToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+ claims.put("roles", List.of("clear-backlog-consumer"));
+ final String consumerToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+
+ @Cleanup
+ PulsarAdmin superUserAdmin = newPulsarAdmin(superUserToken);
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of("clear-backlog-tenant-admin"))
+ .allowedClusters(Sets.newHashSet(configClusterName)).build());
+ superUserAdmin.namespaces().createNamespace(namespace);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, "clear-backlog-consumer",
+ Set.of(AuthAction.consume));
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, subscription, MessageId.earliest);
+ @Cleanup
+ PulsarClient client = newPulsarClient(superUserToken);
+ @Cleanup
+ Producer<byte[]> producer = client.newProducer().topic(topic).enableBatching(false).create();
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Cleanup
+ PulsarAdmin consumerAdmin = newPulsarAdmin(consumerToken);
+ assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> consumerAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, replicatorCursor));
+ assertThrows(PulsarAdminException.NotAuthorizedException.class,
+ () -> consumerAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, bundle,
+ replicatorCursor));
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ numMessages);
+
+ @Cleanup
+ PulsarAdmin tenantAdmin = newPulsarAdmin(tenantAdminToken);
+ tenantAdmin.namespaces().clearNamespaceBacklogForSubscription(namespace, replicatorCursor);
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ 0);
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+ tenantAdmin.namespaces().clearNamespaceBundleBacklogForSubscription(namespace, bundle, replicatorCursor);
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ 0);
+ }
+
+ @Test
+ public void testNamespaceSubscriptionOperationsThroughProxyWithTenantAdminRole() throws Exception {
+ final String tenant = "multi-roles-proxy-tenant";
+ final String namespace = tenant + "/ns";
+ final String topic = "persistent://" + namespace + "/test-topic";
+ final String consumerRole = "proxied-consumer";
+ final String tenantAdminRole = "proxied-tenant-admin";
+ final String subscription = "sub";
+ final String consumerSubscription = consumerRole + "-sub";
+ final String bundle = "0x00000000_0xffffffff";
+ // resolved to the subscription with the same name as the remote cluster part of the name
+ final String replicatorCursor = pulsar.getConfiguration().getReplicatorPrefix() + "." + subscription;
+ final int numMessages = 5;
+ Map<String, Object> claims = new HashMap<>();
+ // the first role is the primary role of the token, the second one gives the proxy tenant admin access
+ claims.put("roles", List.of(PROXY_ROLE, tenantAdminRole));
+ final String proxyToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+
+ @Cleanup
+ PulsarAdmin superUserAdmin = newPulsarAdmin(superUserToken);
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(tenantAdminRole))
+ .allowedClusters(Sets.newHashSet(configClusterName)).build());
+ superUserAdmin.namespaces().createNamespace(namespace);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, consumerRole, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createSubscription(topic, subscription, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(topic, consumerSubscription, MessageId.earliest);
+ @Cleanup
+ PulsarClient client = newPulsarClient(superUserToken);
+ @Cleanup
+ Producer<byte[]> producer = client.newProducer().topic(topic).enableBatching(false).create();
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+ HttpClient httpClient = HttpClient.newHttpClient();
+
+ // the proxy is a tenant admin through a token role, the original principal is an ordinary consumer and is
+ // checked with its own role only
+ for (String bundlePath : new String[]{"", "/" + bundle}) {
+ assertEquals(clearBacklogThroughProxy(httpClient, proxyToken, consumerRole, namespace + bundlePath,
+ replicatorCursor), 403);
+ assertEquals(clearBacklogThroughProxy(httpClient, proxyToken, consumerRole, namespace + bundlePath,
+ subscription), 403);
+ }
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ numMessages);
+
+ // the consumer can clear a subscription that matches its role prefix
+ assertEquals(clearBacklogThroughProxy(httpClient, proxyToken, consumerRole, namespace,
+ consumerSubscription), 204);
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(consumerSubscription)
+ .getMsgBacklog(), 0);
+
+ // a tenant admin forwarded by the proxy keeps the existing behaviour
+ assertEquals(clearBacklogThroughProxy(httpClient, proxyToken, tenantAdminRole, namespace,
+ replicatorCursor), 204);
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ 0);
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+ assertEquals(clearBacklogThroughProxy(httpClient, proxyToken, tenantAdminRole, namespace + "/" + bundle,
+ subscription), 204);
+ assertEquals(superUserAdmin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog(),
+ 0);
+ }
+
+ @Test
+ public void testAllSubscriptionOperationsThroughProxyWithTenantAdminRole() throws Exception {
+ final String tenant = "multi-roles-proxy-all-subscriptions-tenant";
+ final String namespace = tenant + "/ns";
+ final String topic = "persistent://" + namespace + "/topic";
+ final String partitionedTopic = "persistent://" + namespace + "/partitioned";
+ final String partitionedTopic2 = "persistent://" + namespace + "/partitioned-2";
+ final String consumerRole = "proxied-all-subscriptions-consumer";
+ final String tenantAdminRole = "proxied-all-subscriptions-tenant-admin";
+ final String subscription = "sub";
+ final String consumerSubscription = consumerRole + "-sub";
+ final int numMessages = 5;
+ Map<String, Object> claims = new HashMap<>();
+ // the first role is the primary role of the token, the second one gives the proxy tenant admin access
+ claims.put("roles", List.of(PROXY_ROLE, tenantAdminRole));
+ final String proxyToken = Jwts.builder().setClaims(claims).signWith(secretKey).compact();
+
+ @Cleanup
+ PulsarAdmin superUserAdmin = newPulsarAdmin(superUserToken);
+ superUserAdmin.tenants().createTenant(tenant, TenantInfo.builder()
+ .adminRoles(Set.of(tenantAdminRole))
+ .allowedClusters(Sets.newHashSet(configClusterName)).build());
+ superUserAdmin.namespaces().createNamespace(namespace);
+ superUserAdmin.namespaces().grantPermissionOnNamespace(namespace, consumerRole, Set.of(AuthAction.consume));
+ superUserAdmin.namespaces().setSubscriptionAuthMode(namespace, SubscriptionAuthMode.Prefix);
+ superUserAdmin.topics().createNonPartitionedTopic(topic);
+ superUserAdmin.topics().createPartitionedTopic(partitionedTopic, 2);
+ superUserAdmin.topics().createPartitionedTopic(partitionedTopic2, 2);
+ final List<String> topics = List.of(topic, partitionedTopic + "-partition-0",
+ partitionedTopic + "-partition-1", partitionedTopic2 + "-partition-0",
+ partitionedTopic2 + "-partition-1");
+ @Cleanup
+ PulsarClient client = newPulsarClient(superUserToken);
+ for (String t : topics) {
+ superUserAdmin.topics().createSubscription(t, subscription, MessageId.earliest);
+ superUserAdmin.topics().createSubscription(t, consumerSubscription, MessageId.earliest);
+ @Cleanup
+ Producer<byte[]> producer = client.newProducer().topic(t).enableBatching(false).create();
+ for (int i = 0; i < numMessages; i++) {
+ producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ // the messages must be older than the expiry time
+ Thread.sleep(1500);
+ HttpClient httpClient = HttpClient.newHttpClient();
+
+ // the proxy is a tenant admin through a token role, the original principal is an ordinary consumer and
+ // expires only the subscriptions that match its role prefix: on a topic, a partitioned topic and a partition
+ for (String t : List.of(topic, partitionedTopic, partitionedTopic2 + "-partition-0")) {
+ assertEquals(postThroughProxy(httpClient, proxyToken, consumerRole, "/admin/v2/"
+ + TopicName.get(t).getRestPath() + "/all_subscription/expireMessages/1"), 204);
+ }
+ final List<String> expired = topics.subList(0, 4);
+ Awaitility.await().untilAsserted(() -> {
+ for (String t : expired) {
+ assertEquals(getMsgBacklog(superUserAdmin, t, consumerSubscription), 0, t);
+ }
+ });
+ for (String t : topics) {
+ assertEquals(getMsgBacklog(superUserAdmin, t, subscription), numMessages, t);
+ }
+ assertEquals(getMsgBacklog(superUserAdmin, topics.get(4), consumerSubscription), numMessages);
+
+ // the same applies to the namespace clear backlog
+ assertEquals(postThroughProxy(httpClient, proxyToken, consumerRole,
+ "/admin/v2/namespaces/" + namespace + "/clearBacklog"), 204);
+ for (String t : topics) {
+ assertEquals(getMsgBacklog(superUserAdmin, t, consumerSubscription), 0, t);
+ assertEquals(getMsgBacklog(superUserAdmin, t, subscription), numMessages, t);
+ }
+
+ // a tenant admin forwarded by the proxy clears all subscriptions
+ assertEquals(postThroughProxy(httpClient, proxyToken, tenantAdminRole,
+ "/admin/v2/namespaces/" + namespace + "/clearBacklog"), 204);
+ for (String t : topics) {
+ assertEquals(getMsgBacklog(superUserAdmin, t, subscription), 0, t);
+ }
+ }
+
+ private static long getMsgBacklog(PulsarAdmin admin, String topic, String subscription)
+ throws PulsarAdminException {
+ return admin.topics().getStats(topic).getSubscriptions().get(subscription).getMsgBacklog();
+ }
+
+ private int postThroughProxy(HttpClient httpClient, String proxyToken, String originalPrincipal, String path)
+ throws Exception {
+ HttpRequest request = HttpRequest.newBuilder(URI.create(pulsar.getWebServiceAddress() + path))
+ .header("Authorization", "Bearer " + proxyToken)
+ .header("X-Original-Principal", originalPrincipal)
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.noBody())
+ .build();
+ return httpClient.send(request, HttpResponse.BodyHandlers.discarding()).statusCode();
+ }
+
@Test
public void testProxiedRequestsCheckOriginalPrincipalWithItsOwnRole() throws Exception {
final String tenant = "multi-roles-proxied-tenant";
@@ -310,4 +535,16 @@ public class MultiRolesTokenAuthorizationProviderTest extends MockedPulsarServic
.build();
return httpClient.send(request, HttpResponse.BodyHandlers.discarding()).statusCode();
}
+
+ private int clearBacklogThroughProxy(HttpClient httpClient, String proxyToken, String originalPrincipal,
+ String namespaceOrBundle, String subscription) throws Exception {
+ HttpRequest request = HttpRequest.newBuilder(URI.create(pulsar.getWebServiceAddress()
+ + "/admin/v2/namespaces/" + namespaceOrBundle + "/clearBacklog/" + subscription))
+ .header("Authorization", "Bearer " + proxyToken)
+ .header("X-Original-Principal", originalPrincipal)
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.noBody())
+ .build();
+ return httpClient.send(request, HttpResponse.BodyHandlers.discarding()).statusCode();
+ }
}
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
index 47c5fc430f2e..23dbf3e71563 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
@@ -1169,6 +1169,20 @@ public class NamespacesImpl extends BaseResource implements Namespaces {
return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON));
}
+ /**
+ * Used by the broker when dispatching to namespace bundles; not part of the public
+ * {@link org.apache.pulsar.client.admin.Namespaces} API.
+ */
+ public CompletableFuture<Void> clearNamespaceBundleBacklogForSubscriptionAsync(String namespace, String bundle,
+ String subscription, boolean subscriptionOnly) {
+ NamespaceName ns = NamespaceName.get(namespace);
+ WebTarget path = namespacePath(ns, bundle, "clearBacklog", subscription);
+ if (subscriptionOnly) {
+ path = path.queryParam("subscriptionOnly", true);
+ }
+ return asyncPostRequest(path, Entity.entity("", MediaType.APPLICATION_JSON));
+ }
+
@Override
public void unsubscribeNamespace(String namespace, String subscription) throws PulsarAdminException {
sync(() -> unsubscribeNamespaceAsync(namespace, subscription));