You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2018/04/13 18:27:50 UTC
[incubator-pulsar] branch master updated: Compaction CLI
integration test (#1572)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 5c21b33 Compaction CLI integration test (#1572)
5c21b33 is described below
commit 5c21b338b27b043a27d221ad9b252f8d346e786e
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Fri Apr 13 20:27:45 2018 +0200
Compaction CLI integration test (#1572)
Integration test to exercise CLI based compaction from end to end.
---
.../single-cluster-3-bookie-2-broker.yaml | 179 +++++++++++++++++++++
tests/integration/{ => compaction}/pom.xml | 29 ++--
.../pulsar/tests/integration/TestCompaction.java | 103 ++++++++++++
.../src/test/resources/arquillian.xml} | 28 ++--
tests/integration/pom.xml | 1 +
5 files changed, 313 insertions(+), 27 deletions(-)
diff --git a/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml
new file mode 100644
index 0000000..dd12abd
--- /dev/null
+++ b/tests/integration-tests-topologies/src/main/resources/cube-definitions/single-cluster-3-bookie-2-broker.yaml
@@ -0,0 +1,179 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+networks:
+ pulsarnet*:
+ driver: bridge
+
+zookeeper*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env: [ZOOKEEPER_SERVERS=zookeeper]
+ labels:
+ cluster: test
+ service: zookeeper
+ entryPoint: [bin/run-local-zk.sh]
+ aliases:
+ - zookeeper
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+global-zookeeper*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env: [ZOOKEEPER_SERVERS=zookeeper]
+ labels:
+ cluster: test
+ service: global-zookeeper
+ entryPoint: [bin/run-global-zk.sh]
+ aliases:
+ - global-zookeeper
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+init*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env:
+ - cluster=test
+ - zkServers=zookeeper
+ - globalZkServers=global-zookeeper:2184
+ - pulsarNode=pulsar-broker1
+ labels:
+ cluster: test
+ service: init
+ entryPoint: [bin/init-cluster.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.LogToTargetDirStopAction
+ networkMode: pulsarnet*
+
+bookkeeper1*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env:
+ - zkServers=zookeeper
+ - useHostNameAsBookieID=true
+ labels:
+ cluster: test
+ service: bookie
+ entryPoint: [bin/run-bookie.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+bookkeeper2*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env:
+ - zkServers=zookeeper
+ - useHostNameAsBookieID=true
+ labels:
+ cluster: test
+ service: bookie
+ entryPoint: [bin/run-bookie.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+bookkeeper3*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ env:
+ - zkServers=zookeeper
+ - useHostNameAsBookieID=true
+ labels:
+ cluster: test
+ service: bookie
+ entryPoint: [bin/run-bookie.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+pulsar-broker1*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ aliases:
+ - pulsar-broker1
+ env:
+ - zookeeperServers=zookeeper
+ - globalZookeeperServers=global-zookeeper:2184
+ - clusterName=test
+ labels:
+ cluster: test
+ service: pulsar-broker
+ entryPoint: [bin/run-broker.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+pulsar-broker2*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ aliases:
+ - pulsar-broker2
+ env:
+ - zookeeperServers=zookeeper
+ - globalZookeeperServers=global-zookeeper:2184
+ - clusterName=test
+ labels:
+ cluster: test
+ service: pulsar-broker
+ entryPoint: [bin/run-broker.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+pulsar-proxy*:
+ image: apachepulsar/pulsar-test-latest-version:latest
+ await:
+ strategy: org.apache.pulsar.tests.NoopAwaitStrategy
+ aliases:
+ - pulsar-broker2
+ env:
+ - zookeeperServers=zookeeper
+ - globalZookeeperServers=global-zookeeper:2184
+ - clusterName=test
+ labels:
+ cluster: test
+ service: pulsar-proxy
+ entryPoint: [bin/run-proxy.sh]
+ beforeStop:
+ - customBeforeStopAction:
+ strategy: org.apache.pulsar.tests.PulsarLogsToTargetDirStopAction
+ networkMode: pulsarnet*
+
+
diff --git a/tests/integration/pom.xml b/tests/integration/compaction/pom.xml
similarity index 57%
copy from tests/integration/pom.xml
copy to tests/integration/compaction/pom.xml
index a2e36d4..1eee545 100644
--- a/tests/integration/pom.xml
+++ b/tests/integration/compaction/pom.xml
@@ -19,20 +19,29 @@
under the License.
-->
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <packaging>pom</packaging>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.pulsar.tests</groupId>
- <artifactId>tests-parent</artifactId>
+ <artifactId>integration-tests-base</artifactId>
<version>2.0.0-incubating-SNAPSHOT</version>
+ <relativePath>../../integration-tests-base</relativePath>
</parent>
- <groupId>org.apache.pulsar.tests</groupId>
- <artifactId>integration</artifactId>
- <name>Apache Pulsar :: Tests :: Integration</name>
- <modules>
- <module>smoke</module>
- </modules>
+ <groupId>org.apache.pulsar.tests.integration</groupId>
+ <artifactId>compaction</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Pulsar :: Tests :: Integration Tests :: Topic Compaction</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
</project>
diff --git a/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
new file mode 100644
index 0000000..6a6130f
--- /dev/null
+++ b/tests/integration/compaction/src/test/java/org/apache/pulsar/tests/integration/TestCompaction.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.integration;
+
+import com.github.dockerjava.api.DockerClient;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageBuilder;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.tests.DockerUtils;
+import org.apache.pulsar.tests.PulsarClusterUtils;
+
+import org.jboss.arquillian.testng.Arquillian;
+import org.jboss.arquillian.test.api.ArquillianResource;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestCompaction extends Arquillian {
+ private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
+ private static byte[] PASSWD = "foobar".getBytes();
+ private static String clusterName = "test";
+
+ @ArquillianResource
+ DockerClient docker;
+
+ @BeforeMethod
+ public void waitServicesUp() throws Exception {
+ Assert.assertTrue(PulsarClusterUtils.waitZooKeeperUp(docker, clusterName, 30, TimeUnit.SECONDS));
+ Assert.assertTrue(PulsarClusterUtils.waitAllBrokersUp(docker, clusterName));
+ }
+
+ @Test
+ public void testPublishCompactAndConsumeCLI() throws Exception {
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "properties",
+ "create", "compaction-test-cli", "--allowed-clusters", clusterName,
+ "--admin-roles", "admin");
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar-admin", "namespaces",
+ "create", "compaction-test-cli/test/ns1");
+
+ String brokerIp = DockerUtils.getContainerIP(
+ docker, PulsarClusterUtils.proxySet(docker, clusterName).stream().findAny().get());
+ String serviceUrl = "pulsar://" + brokerIp + ":6650";
+ String topic = "persistent://compaction-test-cli/test/ns1/topic1";
+
+ try (PulsarClient client = PulsarClient.create(serviceUrl)) {
+ client.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
+
+ try(Producer<byte[]> producer = client.newProducer().topic(topic).create()) {
+ producer.send(MessageBuilder.create().setKey("key0").setContent("content0".getBytes()).build());
+ producer.send(MessageBuilder.create().setKey("key0").setContent("content1".getBytes()).build());
+ }
+
+ try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+ .readCompacted(true).subscriptionName("sub1").subscribe()) {
+ Message<byte[]> m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content0".getBytes());
+
+ m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content1".getBytes());
+ }
+
+ PulsarClusterUtils.runOnAnyBroker(docker, clusterName,
+ "/pulsar/bin/pulsar", "compact-topic",
+ "-t", topic);
+
+ try (Consumer<byte[]> consumer = client.newConsumer().topic(topic)
+ .readCompacted(true).subscriptionName("sub1").subscribe()) {
+ Message<byte[]> m = consumer.receive();
+ Assert.assertEquals(m.getKey(), "key0");
+ Assert.assertEquals(m.getData(), "content1".getBytes());
+ }
+ }
+ }
+}
diff --git a/tests/integration/pom.xml b/tests/integration/compaction/src/test/resources/arquillian.xml
similarity index 54%
copy from tests/integration/pom.xml
copy to tests/integration/compaction/src/test/resources/arquillian.xml
index a2e36d4..821d6e6 100644
--- a/tests/integration/pom.xml
+++ b/tests/integration/compaction/src/test/resources/arquillian.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
@@ -19,20 +19,14 @@
under the License.
-->
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <packaging>pom</packaging>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.pulsar.tests</groupId>
- <artifactId>tests-parent</artifactId>
- <version>2.0.0-incubating-SNAPSHOT</version>
- </parent>
+<arquillian xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://jboss.org/schema/arquillian"
+ xsi:schemaLocation="http://jboss.org/schema/arquillian
+ http://jboss.org/schema/arquillian/arquillian_1_0.xsd">
- <groupId>org.apache.pulsar.tests</groupId>
- <artifactId>integration</artifactId>
- <name>Apache Pulsar :: Tests :: Integration</name>
- <modules>
- <module>smoke</module>
- </modules>
-</project>
+ <extension qualifier="docker">
+ <property name="definitionFormat">CUBE</property>
+ <property name="dockerContainersResource">cube-definitions/single-cluster-3-bookie-2-broker.yaml</property>
+ </extension>
+
+</arquillian>
diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml
index a2e36d4..4b25ac4 100644
--- a/tests/integration/pom.xml
+++ b/tests/integration/pom.xml
@@ -34,5 +34,6 @@
<name>Apache Pulsar :: Tests :: Integration</name>
<modules>
<module>smoke</module>
+ <module>compaction</module>
</modules>
</project>
--
To stop receiving notification emails like this one, please contact
mmerli@apache.org.