You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/04/17 16:23:25 UTC
[GitHub] merlimat closed pull request #1596: Compaction commands for
pulsar-admin
merlimat closed pull request #1596: Compaction commands for pulsar-admin
URL: https://github.com/apache/incubator-pulsar/pull/1596
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 9ee31acf4..f83cbf5e1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -47,8 +47,11 @@
public static boolean isBatch(RawMessage msg) {
ByteBuf payload = msg.getHeadersAndPayload();
MessageMetadata metadata = Commands.parseMessageMetadata(payload);
- int batchSize = metadata.getNumMessagesInBatch();
- return batchSize > 1;
+ try {
+ return metadata.hasNumMessagesInBatch();
+ } finally {
+ metadata.recycle();
+ }
}
public static List<ImmutablePair<MessageId,String>> extractIdsAndKeys(RawMessage msg)
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
index 2eaa8d0c9..c263b1c28 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
@@ -312,10 +312,14 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map<String, MessageId>
private static Pair<String,Integer> extractKeyAndSize(RawMessage m) {
ByteBuf headersAndPayload = m.getHeadersAndPayload();
MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload);
- if (msgMetadata.hasPartitionKey()) {
- return Pair.of(msgMetadata.getPartitionKey(), headersAndPayload.readableBytes());
- } else {
- return null;
+ try {
+ if (msgMetadata.hasPartitionKey()) {
+ return Pair.of(msgMetadata.getPartitionKey(), headersAndPayload.readableBytes());
+ } else {
+ return null;
+ }
+ } finally {
+ msgMetadata.recycle();
}
}
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
index 327e2105b..dfc72e227 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdPersistentTopics.java
@@ -30,6 +30,7 @@
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.impl.BatchMessageIdImpl;
import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.apache.pulsar.common.compaction.CompactionStatus;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@@ -78,6 +79,8 @@ public CmdPersistentTopics(PulsarAdmin admin) {
jcommander.addCommand("peek-messages", new PeekMessages());
jcommander.addCommand("reset-cursor", new ResetCursor());
jcommander.addCommand("terminate", new Terminate());
+ jcommander.addCommand("compact", new Compact());
+ jcommander.addCommand("compaction-status", new CompactionStatusCmd());
}
@Parameters(commandDescription = "Get the list of topics under a namespace.")
@@ -540,6 +543,61 @@ void run() throws PulsarAdminException {
}
}
+ @Parameters(commandDescription = "Compact a topic")
+ private class Compact extends CliCommand {
+ @Parameter(description = "persistent://property/cluster/namespace/topic", required = true)
+ private java.util.List<String> params;
+
+ @Override
+ void run() throws PulsarAdminException {
+ String persistentTopic = validatePersistentTopic(params);
+
+ persistentTopics.triggerCompaction(persistentTopic);
+ System.out.println("Topic compaction requested for " + persistentTopic);
+ }
+ }
+
+ @Parameters(commandDescription = "Status of compaction on a topic")
+ private class CompactionStatusCmd extends CliCommand {
+ @Parameter(description = "persistent://property/cluster/namespace/topic", required = true)
+ private java.util.List<String> params;
+
+ @Parameter(names = { "-w", "--wait-complete" },
+ description = "Wait for compaction to complete", required = false)
+ private boolean wait = false;
+
+ @Override
+ void run() throws PulsarAdminException {
+ String persistentTopic = validatePersistentTopic(params);
+
+ try {
+ CompactionStatus status = persistentTopics.compactionStatus(persistentTopic);
+ while (wait && status.status == CompactionStatus.Status.RUNNING) {
+ Thread.sleep(1000);
+ status = persistentTopics.compactionStatus(persistentTopic);
+ }
+
+ switch (status.status) {
+ case NOT_RUN:
+ System.out.println("Compaction has not been run for " + persistentTopic
+ + " since broker startup");
+ break;
+ case RUNNING:
+ System.out.println("Compaction is currently running");
+ break;
+ case SUCCESS:
+ System.out.println("Compaction was a success");
+ break;
+ case ERROR:
+ System.out.println("Error in compaction");
+ throw new PulsarAdminException("Error compacting: " + status.lastError);
+ }
+ } catch (InterruptedException e) {
+ throw new PulsarAdminException(e);
+ }
+ }
+ }
+
private static int validateTimeString(String s) {
char last = s.charAt(s.length() - 1);
String subStr = s.substring(0, s.length() - 1);
diff --git a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
index 3611736b3..bd849133f 100644
--- a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
+++ b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
@@ -56,21 +56,20 @@ public void testPublishCompactAndConsumeCLI() throws Exception {
"--admin-roles", "admin");
PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
"/pulsar/bin/pulsar-admin", "namespaces",
- "create", "compaction-test-cli/test/ns1");
+ "create", "compaction-test-cli/ns1");
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "namespaces",
+ "set-clusters", "--clusters", "test", "compaction-test-cli/ns1");
String brokerIp = DockerUtils.getContainerIP(
docker, PulsarClusterUtils.proxySet(docker, clusterName).stream().findAny().get());
String serviceUrl = "pulsar://" + brokerIp + ":6650";
- String topic = "persistent://compaction-test-cli/test/ns1/topic1";
+ String topic = "persistent://compaction-test-cli/ns1/topic1";
try (PulsarClient client = PulsarClient.builder().serviceUrl(serviceUrl).build()) {
client.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
- try(Producer<byte[]> producer = client.newProducer()
- .topic(topic)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create()) {
+ try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) {
producer.send(MessageBuilder.create().setKey("key0").setContent("content0".getBytes()).build());
producer.send(MessageBuilder.create().setKey("key0").setContent("content1".getBytes()).build());
}
@@ -98,4 +97,57 @@ public void testPublishCompactAndConsumeCLI() throws Exception {
}
}
}
+
+ @Test
+ public void testPublishCompactAndConsumeRest() throws Exception {
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "tenants",
+ "create", "compaction-test-rest", "--allowed-clusters", clusterName,
+ "--admin-roles", "admin");
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "namespaces",
+ "create", "compaction-test-rest/ns1");
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "namespaces",
+ "set-clusters", "--clusters", "test", "compaction-test-rest/ns1");
+
+ String brokerIp = DockerUtils.getContainerIP(
+ docker, PulsarClusterUtils.proxySet(docker, clusterName).stream().findAny().get());
+ String serviceUrl = "pulsar://" + brokerIp + ":6650";
+ String topic = "persistent://compaction-test-rest/ns1/topic1";
+
+ try (PulsarClient client = PulsarClient.create(serviceUrl)) {
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
+
+ try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) {
+ producer.send(MessageBuilder.create().setKey("key0").setContent("content0".getBytes()).build());
+ producer.send(MessageBuilder.create().setKey("key0").setContent("content1".getBytes()).build());
+ }
+
+ try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+ .readCompacted(true).subscriptionName("sub1").subscribe()) {
+ Message<byte[]> m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content0".getBytes());
+
+ m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content1".getBytes());
+ }
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "persistent", "compact", topic);
+
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "persistent", "compaction-status",
+ "-w", topic);
+
+ try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+ .readCompacted(true).subscriptionName("sub1").subscribe()) {
+ Message<byte[]> m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content1".getBytes());
+ }
+ }
+ }
+
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services