You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by si...@apache.org on 2018/06/09 19:41:44 UTC

[incubator-pulsar] branch master updated: Rest API, AdminClient and CLI for compaction threshold (#1924)

This is an automated email from the ASF dual-hosted git repository.

sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 2245c37  Rest API, AdminClient and CLI for compaction threshold (#1924)
2245c37 is described below

commit 2245c3785e161d3d421bdd2120e0272878b01c6f
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Sat Jun 9 21:41:39 2018 +0200

    Rest API, AdminClient and CLI for compaction threshold (#1924)
    
    Allow the user to specify a namespace policy for the maximum backlog
    size for topics in that namespace. When a topic backlog reaches this
    size, compaction is triggered.
---
 .../pulsar/broker/admin/impl/NamespacesBase.java   | 41 ++++++++++++++++
 .../apache/pulsar/broker/admin/v1/Namespaces.java  | 30 ++++++++++++
 .../apache/pulsar/broker/admin/v2/Namespaces.java  | 29 +++++++++++
 .../org/apache/pulsar/client/admin/Namespaces.java | 47 ++++++++++++++++++
 .../client/admin/internal/NamespacesImpl.java      | 22 +++++++++
 .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 33 +++++++++++++
 .../single-cluster-3-bookie-2-broker.yaml          |  2 +
 .../pulsar/tests/integration/TestCompaction.java   | 57 ++++++++++++++++++++++
 8 files changed, 261 insertions(+)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
index e11864d..c845fd5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java
@@ -1529,5 +1529,46 @@ public abstract class NamespacesBase extends AdminResource {
         }
     }
 
+    protected long internalGetCompactionThreshold() {
+        validateAdminAccessForTenant(namespaceName.getTenant());
+        return getNamespacePolicies(namespaceName).compaction_threshold;
+    }
+
+    protected void internalSetCompactionThreshold(long newThreshold) {
+        validateSuperUserAccess();
+        validatePoliciesReadOnlyAccess();
+
+        try {
+            Stat nodeStat = new Stat();
+            final String path = path(POLICIES, namespaceName.toString());
+            byte[] content = globalZk().getData(path, null, nodeStat);
+            Policies policies = jsonMapper().readValue(content, Policies.class);
+            if (newThreshold < 0) {
+                throw new RestException(Status.PRECONDITION_FAILED,
+                        "compactionThreshold must be 0 or more");
+            }
+            policies.compaction_threshold = newThreshold;
+            globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion());
+            policiesCache().invalidate(path(POLICIES, namespaceName.toString()));
+            log.info("[{}] Successfully updated compactionThreshold configuration: namespace={}, value={}",
+                     clientAppId(), namespaceName, policies.compaction_threshold);
+
+        } catch (KeeperException.NoNodeException e) {
+            log.warn("[{}] Failed to update compactionThreshold configuration for namespace {}: does not exist",
+                     clientAppId(), namespaceName);
+            throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
+        } catch (KeeperException.BadVersionException e) {
+            log.warn("[{}] Failed to update maxConsumersPerSubscription configuration for namespace {}: concurrent modification",
+                     clientAppId(), namespaceName);
+            throw new RestException(Status.CONFLICT, "Concurrent modification");
+        } catch (RestException pfe) {
+            throw pfe;
+        } catch (Exception e) {
+            log.error("[{}] Failed to update compactionThreshold configuration for namespace {}",
+                      clientAppId(), namespaceName, e);
+            throw new RestException(e);
+        }
+    }
+
     private static final Logger log = LoggerFactory.getLogger(NamespacesBase.class);
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java
index cd34b0d..f8edcf1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Namespaces.java
@@ -715,6 +715,36 @@ public class Namespaces extends NamespacesBase {
         internalSetMaxConsumersPerSubscription(maxConsumersPerSubscription);
     }
 
+    @GET
+    @Path("/{property}/{cluster}/{namespace}/compactionThreshold")
+    @ApiOperation(value = "Maximum number of uncompacted bytes in topics before compaction is triggered.",
+                  notes = "The backlog size is compared to the threshold periodically. "
+                          + "A threshold of 0 disabled automatic compaction")
+    @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
+                            @ApiResponse(code = 404, message = "Namespace doesn't exist") })
+    public long getCompactionThreshold(@PathParam("property") String property,
+                                       @PathParam("cluster") String cluster,
+                                       @PathParam("namespace") String namespace) {
+        validateNamespaceName(property, cluster, namespace);
+        return internalGetCompactionThreshold();
+    }
+
+    @PUT
+    @Path("/{property}/{cluster}/{namespace}/compactionThreshold")
+    @ApiOperation(value = "Set maximum number of uncompacted bytes in a topic before compaction is triggered.",
+                  notes = "The backlog size is compared to the threshold periodically. "
+                          + "A threshold of 0 disabled automatic compaction")
+    @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
+                            @ApiResponse(code = 404, message = "Namespace doesn't exist"),
+                            @ApiResponse(code = 409, message = "Concurrent modification"),
+                            @ApiResponse(code = 412, message = "compactionThreshold value is not valid") })
+    public void setCompactionThreshold(@PathParam("property") String property,
+                                       @PathParam("cluster") String cluster,
+                                       @PathParam("namespace") String namespace,
+                                       long newThreshold) {
+        validateNamespaceName(property, cluster, namespace);
+        internalSetCompactionThreshold(newThreshold);
+    }
 
     private static final Logger log = LoggerFactory.getLogger(Namespaces.class);
 }
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
index 15ac532..bd467ed 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java
@@ -657,5 +657,34 @@ public class Namespaces extends NamespacesBase {
         return policies;
     }
 
+    @GET
+    @Path("/{property}/{namespace}/compactionThreshold")
+    @ApiOperation(value = "Maximum number of uncompacted bytes in topics before compaction is triggered.",
+                  notes = "The backlog size is compared to the threshold periodically. "
+                          + "A threshold of 0 disabled automatic compaction")
+    @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
+                            @ApiResponse(code = 404, message = "Namespace doesn't exist") })
+    public long getCompactionThreshold(@PathParam("property") String property,
+                                       @PathParam("namespace") String namespace) {
+        validateNamespaceName(property, namespace);
+        return internalGetCompactionThreshold();
+    }
+
+    @PUT
+    @Path("/{property}/{namespace}/compactionThreshold")
+    @ApiOperation(value = "Set maximum number of uncompacted bytes in a topic before compaction is triggered.",
+                  notes = "The backlog size is compared to the threshold periodically. "
+                          + "A threshold of 0 disabled automatic compaction")
+    @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
+                            @ApiResponse(code = 404, message = "Namespace doesn't exist"),
+                            @ApiResponse(code = 409, message = "Concurrent modification"),
+                            @ApiResponse(code = 412, message = "compactionThreshold value is not valid") })
+    public void setCompactionThreshold(@PathParam("property") String property,
+                                       @PathParam("namespace") String namespace,
+                                       long newThreshold) {
+        validateNamespaceName(property, namespace);
+        internalSetCompactionThreshold(newThreshold);
+    }
+
     private static final Logger log = LoggerFactory.getLogger(Namespaces.class);
 }
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java
index 373c535..03cb52d 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java
@@ -1117,4 +1117,51 @@ public interface Namespaces {
      *             Unexpected error
      */
     void setMaxConsumersPerSubscription(String namespace, int maxConsumersPerSubscription) throws PulsarAdminException;
+
+    /**
+     * Get the compactionThreshold for a namespace. The maximum number of bytes topics in the namespace
+     * can have before compaction is triggered. 0 disables.
+     * <p>
+     * Response example:
+     *
+     * <pre>
+     * <code>10000000</code>
+     * </pre>
+     *
+     * @param namespace
+     *            Namespace name
+     *
+     * @throws NotAuthorizedException
+     *             Don't have admin permission
+     * @throws NotFoundException
+     *             Namespace does not exist
+     * @throws PulsarAdminException
+     *             Unexpected error
+     */
+    long getCompactionThreshold(String namespace) throws PulsarAdminException;
+
+    /**
+     * Set the compactionThreshold for a namespace. The maximum number of bytes topics in the namespace
+     * can have before compaction is triggered. 0 disables.
+     * <p>
+     * Request example:
+     *
+     * <pre>
+     * <code>10000000</code>
+     * </pre>
+     *
+     * @param namespace
+     *            Namespace name
+     * @param compactionThreshold
+     *            maximum number of backlog bytes before compaction is triggered
+     *
+     * @throws NotAuthorizedException
+     *             Don't have admin permission
+     * @throws NotFoundException
+     *             Namespace does not exist
+     * @throws PulsarAdminException
+     *             Unexpected error
+     */
+    void setCompactionThreshold(String namespace, long compactionThreshold) throws PulsarAdminException;
+
 }
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
index 7c4fd04..6c79a06 100644
--- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
+++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java
@@ -657,6 +657,28 @@ public class NamespacesImpl extends BaseResource implements Namespaces {
         }
     }
 
+    @Override
+    public long getCompactionThreshold(String namespace) throws PulsarAdminException {
+        try {
+            NamespaceName ns = NamespaceName.get(namespace);
+            WebTarget path = namespacePath(ns, "compactionThreshold");
+            return request(path).get(Long.class);
+        } catch (Exception e) {
+            throw getApiException(e);
+        }
+    }
+
+    @Override
+    public void setCompactionThreshold(String namespace, long compactionThreshold) throws PulsarAdminException {
+        try {
+            NamespaceName ns = NamespaceName.get(namespace);
+            WebTarget path = namespacePath(ns, "compactionThreshold");
+            request(path).put(Entity.entity(compactionThreshold, MediaType.APPLICATION_JSON), ErrorData.class);
+        } catch (Exception e) {
+            throw getApiException(e);
+        }
+    }
+
     private WebTarget namespacePath(NamespaceName namespace, String... parts) {
         final WebTarget base = namespace.isV2() ? adminV2Namespaces : adminNamespaces;
         WebTarget namespacePath = base.path(namespace.toString());
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
index 5259879..9603247 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
@@ -766,6 +766,36 @@ public class CmdNamespaces extends CmdBase {
         }
     }
 
+    @Parameters(commandDescription = "Get compactionThreshold for a namespace")
+    private class GetCompactionThreshold extends CliCommand {
+        @Parameter(description = "tenant/namespace\n", required = true)
+        private java.util.List<String> params;
+
+        @Override
+        void run() throws PulsarAdminException {
+            String namespace = validateNamespace(params);
+            print(admin.namespaces().getCompactionThreshold(namespace));
+        }
+    }
+
+    @Parameters(commandDescription = "Set compactionThreshold for a namespace")
+    private class SetCompactionThreshold extends CliCommand {
+        @Parameter(description = "tenant/namespace", required = true)
+        private java.util.List<String> params;
+
+        @Parameter(names = { "--threshold", "-t" },
+                   description = "Maximum number of bytes in a topic backlog before compaction is triggered "
+                                 + "(eg: 10M, 16G, 3T). 0 disables automatic compaction",
+                   required = true)
+        private String threshold = "0";
+
+        @Override
+        void run() throws PulsarAdminException {
+            String namespace = validateNamespace(params);
+            admin.namespaces().setCompactionThreshold(namespace, validateSizeString(threshold));
+        }
+    }
+
     private static long validateSizeString(String s) {
         char last = s.charAt(s.length() - 1);
         String subStr = s.substring(0, s.length() - 1);
@@ -874,5 +904,8 @@ public class CmdNamespaces extends CmdBase {
         jcommander.addCommand("set-max-consumers-per-topic", new SetMaxConsumersPerTopic());
         jcommander.addCommand("get-max-consumers-per-subscription", new GetMaxConsumersPerSubscription());
         jcommander.addCommand("set-max-consumers-per-subscription", new SetMaxConsumersPerSubscription());
+
+        jcommander.addCommand("get-compaction-threshold", new GetCompactionThreshold());
+        jcommander.addCommand("set-compaction-threshold", new SetCompactionThreshold());
     }
 }
diff --git a/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml
index 1a20574..268e6d7 100644
--- a/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml
+++ b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml
@@ -131,6 +131,7 @@ pulsar-broker1*:
     - zookeeperServers=zookeeper
     - configurationStoreServers=configuration-store:2184
     - clusterName=test
+    - brokerServiceCompactionMonitorIntervalInSeconds=1
   labels:
     cluster: test
     service: pulsar-broker
@@ -150,6 +151,7 @@ pulsar-broker2*:
     - zookeeperServers=zookeeper
     - configurationStoreServers=configuration-store:2184
     - clusterName=test
+    - brokerServiceCompactionMonitorIntervalInSeconds=1
   labels:
     cluster: test
     service: pulsar-broker
diff --git a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
index 5570f05..a716f9d 100644
--- a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
+++ b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
@@ -150,4 +150,61 @@ public class TestCompaction extends Arquillian {
         }
     }
 
+    private static void waitAndVerifyCompacted(PulsarClient client, String topic,
+                                               String sub, String expectedKey, String expectedValue) throws Exception {
+        for (int i = 0; i < 60; i++) {
+            try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+                 .readCompacted(true).subscriptionName(sub).subscribe()) {
+                Message<byte[]> m = consumer.receive();
+                Assert.assertEquals(m.getKey(), expectedKey);
+                if (new String(m.getData()).equals(expectedValue)) {
+                    break;
+                }
+            }
+            Thread.sleep(1000);
+        }
+        try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+                .readCompacted(true).subscriptionName(sub).subscribe()) {
+            Message<byte[]> m = consumer.receive();
+            Assert.assertEquals(m.getKey(), expectedKey);
+            Assert.assertEquals(new String(m.getData()), expectedValue);
+        }
+    }
+
+    @Test
+    public void testPublishWithAutoCompaction() throws Exception {
+        PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+                                          PulsarClusterUtils.PULSAR_ADMIN, "tenants",
+                                          "create", "compaction-test-auto",
+                                          "--allowed-clusters", clusterName,
+                                          "--admin-roles", "admin");
+        PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+                PulsarClusterUtils.PULSAR_ADMIN, "namespaces",
+                "create", "--clusters", "test", "compaction-test-auto/ns1");
+        PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+                PulsarClusterUtils.PULSAR_ADMIN, "namespaces",
+                "set-compaction-threshold", "--threshold", "1", "compaction-test-auto/ns1");
+
+        String brokerIp = DockerUtils.getContainerIP(
+                docker, PulsarClusterUtils.proxySet(docker, clusterName).stream().findAny().get());
+        String serviceUrl = "pulsar://" + brokerIp + ":6650";
+        String topic = "persistent://compaction-test-auto/ns1/topic1";
+
+        try (PulsarClient client = PulsarClient.create(serviceUrl)) {
+            client.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
+
+            try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) {
+                producer.send(MessageBuilder.create().setKey("key0").setContent("content0".getBytes()).build());
+                producer.send(MessageBuilder.create().setKey("key0").setContent("content1".getBytes()).build());
+            }
+
+            waitAndVerifyCompacted(client, topic, "sub1", "key0", "content1");
+
+            try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) {
+                producer.send(MessageBuilder.create().setKey("key0").setContent("content2".getBytes()).build());
+            }
+            waitAndVerifyCompacted(client, topic, "sub1", "key0", "content2");
+        }
+    }
+
 }

-- 
To stop receiving notification emails like this one, please contact
sijie@apache.org.