You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by pe...@apache.org on 2020/11/09 00:28:59 UTC
[pulsar] branch master updated: PIP-62: Delete pulsar adapters
(#8480)
This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new c43be24 PIP-62: Delete pulsar adapters (#8480)
c43be24 is described below
commit c43be243892adf7266428709721e57e3cecd19b6
Author: Sijie Guo <si...@apache.org>
AuthorDate: Sun Nov 8 17:28:34 2020 -0700
PIP-62: Delete pulsar adapters (#8480)
Delete the pulsar adapter modules because they are already moved to https://github.com/apache/pulsar-adapters
---
.github/workflows/ci-unit.yaml | 4 -
pom.xml | 5 -
pulsar-client-kafka-compat/pom.xml | 62 --
.../pulsar-client-kafka-shaded/pom.xml | 255 -------
.../pulsar-client-kafka-shaded_0_8/pom.xml | 413 ------------
.../pulsar-client-kafka-shaded_0_9/pom.xml | 251 -------
.../pulsar-client-kafka-tests/pom.xml | 78 ---
.../kafka/compat/examples/ConsumerAvroExample.java | 75 ---
.../kafka/compat/examples/ConsumerExample.java | 60 --
.../kafka/compat/examples/ProducerAvroExample.java | 69 --
.../kafka/compat/examples/ProducerExample.java | 53 --
.../client/kafka/compat/examples/utils/Bar.java | 30 -
.../client/kafka/compat/examples/utils/Foo.java | 35 -
.../pulsar-client-kafka-tests_0_8/pom.xml | 84 ---
.../compat/examples/HighLevelConsumerExample.java | 121 ----
.../compat/examples/LowLevelConsumerExample.java | 146 ----
.../kafka/compat/examples/ProducerExample.java | 131 ----
.../client/kafka/compat/examples/utils/Tweet.java | 67 --
.../pulsar-client-kafka-tests_0_9/pom.xml | 77 ---
.../kafka/compat/examples/ConsumerAvroExample.java | 75 ---
.../kafka/compat/examples/ConsumerExample.java | 60 --
.../kafka/compat/examples/ProducerAvroExample.java | 69 --
.../kafka/compat/examples/ProducerExample.java | 52 --
.../client/kafka/compat/examples/utils/Bar.java | 30 -
.../client/kafka/compat/examples/utils/Foo.java | 35 -
.../pulsar-client-kafka/pom.xml | 72 --
.../clients/consumer/PulsarKafkaConsumer.java | 743 ---------------------
.../clients/producer/PulsarKafkaProducer.java | 373 -----------
.../client/kafka/compat/KafkaMessageRouter.java | 44 --
.../compat/KafkaProducerInterceptorWrapper.java | 270 --------
.../kafka/compat/PulsarClientKafkaConfig.java | 124 ----
.../kafka/compat/PulsarConsumerKafkaConfig.java | 72 --
.../client/kafka/compat/PulsarKafkaSchema.java | 82 ---
.../kafka/compat/PulsarProducerKafkaConfig.java | 71 --
.../clients/producer/PulsarKafkaProducerTest.java | 274 --------
.../KafkaProducerInterceptorWrapperTest.java | 158 -----
.../pulsar-client-kafka_0_8/pom.xml | 125 ----
.../kafka/clients/consumer/ConsumerConnector.java | 230 -------
.../kafka/clients/consumer/ConsumerIterator.java | 143 ----
.../clients/consumer/PulsarConsumerConfig.java | 28 -
.../clients/consumer/PulsarKafkaConsumer.java | 30 -
.../kafka/clients/consumer/PulsarKafkaStream.java | 68 --
.../clients/consumer/PulsarMessageAndMetadata.java | 92 ---
.../clients/producer/PulsarClientKafkaConfig.java | 169 -----
.../clients/producer/PulsarKafkaProducer.java | 303 ---------
.../consumer/PulsarByteBufferMessageSet.java | 93 ---
.../simple/consumer/PulsarFetchResponse.java | 47 --
.../simple/consumer/PulsarKafkaSimpleConsumer.java | 354 ----------
.../clients/simple/consumer/PulsarMessage.java | 41 --
.../simple/consumer/PulsarMsgAndOffset.java | 40 --
.../simple/consumer/PulsarOffsetCommitRequest.java | 61 --
.../consumer/PulsarOffsetCommitResponse.java | 43 --
.../simple/consumer/PulsarOffsetFetchRequest.java | 36 -
.../simple/consumer/PulsarOffsetFetchResponse.java | 40 --
.../consumer/PulsarOffsetMetadataAndError.java | 44 --
.../simple/consumer/PulsarOffsetRequest.java | 40 --
.../simple/consumer/PulsarOffsetResponse.java | 52 --
.../simple/consumer/PulsarPartitionMetadata.java | 51 --
.../simple/consumer/PulsarTopicMetadata.java | 42 --
.../consumer/PulsarTopicMetadataResponse.java | 71 --
.../clients/consumer/PulsarKafkaConsumerTest.java | 75 ---
.../clients/producer/PulsarKafkaProducerTest.java | 124 ----
.../kafka/test/KafkaProducerConsumerTest.java | 247 -------
.../test/KafkaProducerSimpleConsumerTest.java | 257 -------
.../pulsar-client-kafka_0_9/pom.xml | 71 --
.../clients/consumer/PulsarKafkaConsumer.java | 596 -----------------
.../clients/producer/PulsarKafkaProducer.java | 330 ---------
.../client/kafka/compat/KafkaMessageRouter.java | 44 --
.../kafka/compat/PulsarClientKafkaConfig.java | 117 ----
.../kafka/compat/PulsarConsumerKafkaConfig.java | 72 --
.../client/kafka/compat/PulsarKafkaSchema.java | 82 ---
.../kafka/compat/PulsarProducerKafkaConfig.java | 65 --
.../clients/producer/PulsarKafkaProducerTest.java | 192 ------
pulsar-flink/README.md | 27 -
pulsar-flink/pom.xml | 169 -----
.../connectors/pulsar/BasePulsarOutputFormat.java | 133 ----
.../connectors/pulsar/PulsarAvroOutputFormat.java | 44 --
.../connectors/pulsar/PulsarCsvOutputFormat.java | 45 --
.../connectors/pulsar/PulsarJsonOutputFormat.java | 43 --
.../connectors/pulsar/PulsarOutputFormat.java | 49 --
.../batch/connectors/pulsar/package-info.java | 22 -
.../serialization/AvroSerializationSchema.java | 60 --
.../serialization/CsvSerializationSchema.java | 53 --
.../serialization/JsonSerializationSchema.java | 41 --
.../pulsar/serialization/package-info.java | 22 -
.../connectors/pulsar/CachedPulsarClient.java | 110 ---
.../connectors/pulsar/FlinkPulsarProducer.java | 351 ----------
.../connectors/pulsar/PulsarAvroTableSink.java | 197 ------
.../connectors/pulsar/PulsarConsumerSource.java | 200 ------
.../connectors/pulsar/PulsarJsonTableSink.java | 69 --
.../connectors/pulsar/PulsarProduceMode.java | 37 -
.../connectors/pulsar/PulsarSourceBase.java | 31 -
.../connectors/pulsar/PulsarSourceBuilder.java | 332 ---------
.../connectors/pulsar/PulsarTableSink.java | 178 -----
.../streaming/connectors/pulsar/package-info.java | 22 -
.../pulsar/partitioner/PulsarKeyExtractor.java | 38 --
.../partitioner/PulsarPropertiesExtractor.java | 40 --
.../pulsar/partitioner/package-info.java | 22 -
.../pulsar/PulsarAvroOutputFormatTest.java | 117 ----
.../pulsar/PulsarCsvOutputFormatTest.java | 116 ----
.../pulsar/PulsarJsonOutputFormatTest.java | 116 ----
.../connectors/pulsar/PulsarOutputFormatTest.java | 195 ------
.../serialization/AvroSerializationSchemaTest.java | 66 --
.../serialization/CsvSerializationSchemaTest.java | 53 --
.../serialization/JsonSerializationSchemaTest.java | 94 ---
.../connectors/pulsar/CachedPulsarClientTest.java | 128 ----
.../connectors/pulsar/PulsarAvroTableSinkTest.java | 122 ----
.../pulsar/PulsarConsumerSourceTests.java | 631 -----------------
.../connectors/pulsar/PulsarJsonTableSinkTest.java | 118 ----
.../connectors/pulsar/PulsarSourceBuilderTest.java | 237 -------
.../src/test/resources/avro/NasaMission.avsc | 10 -
pulsar-log4j2-appender/pom.xml | 74 --
.../pulsar/log4j2/appender/PulsarAppender.java | 212 ------
.../pulsar/log4j2/appender/PulsarManager.java | 142 ----
.../pulsar/log4j2/appender/PulsarAppenderTest.java | 240 -------
.../builder/ConfigurationAssemblerTest.java | 115 ----
.../appender/builder/ConfigurationBuilderTest.java | 120 ----
.../builder/CustomConfigurationFactory.java | 86 ---
.../src/test/resources/PulsarAppenderTest.xml | 49 --
pulsar-spark/pom.xml | 106 ---
.../pulsar/spark/SparkStreamingPulsarReceiver.java | 105 ---
pulsar-storm/pom.xml | 76 ---
.../apache/pulsar/storm/MessageToValuesMapper.java | 44 --
.../java/org/apache/pulsar/storm/PulsarBolt.java | 209 ------
.../pulsar/storm/PulsarBoltConfiguration.java | 57 --
.../java/org/apache/pulsar/storm/PulsarSpout.java | 485 --------------
.../pulsar/storm/PulsarSpoutConfiguration.java | 195 ------
.../apache/pulsar/storm/PulsarSpoutConsumer.java | 58 --
.../pulsar/storm/PulsarStormConfiguration.java | 90 ---
.../java/org/apache/pulsar/storm/PulsarTuple.java | 45 --
.../apache/pulsar/storm/SharedPulsarClient.java | 153 -----
.../apache/pulsar/storm/TupleToMessageMapper.java | 66 --
pulsar-storm/src/main/javadoc/overview.html | 29 -
.../org/apache/pulsar/storm/PulsarSpoutTest.java | 178 -----
134 files changed, 16407 deletions(-)
diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml
index 208b611..3de95d8 100644
--- a/.github/workflows/ci-unit.yaml
+++ b/.github/workflows/ci-unit.yaml
@@ -64,10 +64,6 @@ jobs:
if: steps.docs.outputs.changed_only == 'no'
run: ./build/retry.sh mvn -B -ntp test -pl managed-ledger -Dinclude="**/ManagedLedgerTest.java" -DtestForkCount=1 -DtestReuseFork=true
- - name: run flaky test "**/KafkaProducerSimpleConsumerTest.java"
- if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -ntp test -pl pulsar-client-kafka-compat/pulsar-client-kafka_0_8 -Dinclude="**/KafkaProducerSimpleConsumerTest.java" -DtestForkCount=1 -DtestReuseFork=true
-
- name: run flaky test "**/TestPulsarKeyValueSchemaHandler.java"
if: steps.docs.outputs.changed_only == 'no'
run: ./build/retry.sh mvn -B -ntp test -pl pulsar-sql/presto-pulsar-plugin -Dinclude="**/TestPulsarKeyValueSchemaHandler.java" -DtestForkCount=1 -DtestReuseFork=true
diff --git a/pom.xml b/pom.xml
index dc17858..7034bd9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1579,16 +1579,11 @@ flexible messaging model and an intuitive client API.</description>
<module>pulsar-websocket</module>
<module>pulsar-proxy</module>
<module>pulsar-discovery-service</module>
- <module>pulsar-storm</module>
- <module>pulsar-flink</module>
- <module>pulsar-spark</module>
<module>pulsar-zookeeper-utils</module>
<module>pulsar-testclient</module>
<module>pulsar-broker-auth-athenz</module>
<module>pulsar-client-auth-athenz</module>
- <module>pulsar-client-kafka-compat</module>
<module>pulsar-zookeeper</module>
- <module>pulsar-log4j2-appender</module>
<module>pulsar-sql</module>
<module>dashboard</module>
<module>pulsar-broker-auth-sasl</module>
diff --git a/pulsar-client-kafka-compat/pom.xml b/pulsar-client-kafka-compat/pom.xml
deleted file mode 100644
index 2732397..0000000
--- a/pulsar-client-kafka-compat/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <name>Pulsar Kafka compatibility</name>
-
- <packaging>pom</packaging>
-
- <modules>
- <module>pulsar-client-kafka</module>
- <module>pulsar-client-kafka_0_8</module>
- <module>pulsar-client-kafka_0_9</module>
- <module>pulsar-client-kafka-shaded</module>
- <module>pulsar-client-kafka-shaded_0_8</module>
- <module>pulsar-client-kafka-shaded_0_9</module>
- <module>pulsar-client-kafka-tests</module>
- <module>pulsar-client-kafka-tests_0_8</module>
- <module>pulsar-client-kafka-tests_0_9</module>
- </modules>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml
deleted file mode 100644
index 463b496..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml
+++ /dev/null
@@ -1,255 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka</artifactId>
- <name>Pulsar Kafka compatibility :: API</name>
-
- <description>Drop-in replacement for Kafka client library that publishes and consumes
- messages on Pulsar topics</description>
-
- <properties>
- <kafka.version>0.10.2.1</kafka.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka-original</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <artifactSet>
- <includes>
- <include>org.apache.kafka:kafka-clients</include>
- <include>org.apache.pulsar:pulsar-client-kafka-original</include>
- <include>org.apache.pulsar:pulsar-client-original</include>
- <include>org.apache.commons:commons-lang3</include>
- <include>commons-codec:commons-codec</include>
- <include>org.apache.bookkeeper:bookkeeper-common-allocator</include>
- <include>commons-collections:commons-collections</include>
- <include>org.asynchttpclient:*</include>
- <include>io.netty:netty-codec-http</include>
- <include>io.netty:netty-transport-native-epoll</include>
- <include>org.reactivestreams:reactive-streams</include>
- <include>com.typesafe.netty:netty-reactive-streams</include>
- <include>org.javassist:javassist</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>com.google.guava:guava</include>
- <include>com.google.code.gson:gson</include>
- <include>com.fasterxml.jackson.core</include>
- <include>com.fasterxml.jackson.module</include>
- <include>com.fasterxml.jackson.dataformat</include>
- <include>io.netty:netty</include>
- <include>io.netty:netty-*</include>
- <include>org.apache.pulsar:pulsar-common</include>
- <include>org.apache.bookkeeper:circe-checksum</include>
- <include>com.yahoo.datasketches:sketches-core</include>
- <include>org.eclipse.jetty:*</include>
- <include>com.yahoo.datasketches:*</include>
- <include>commons-*:*</include>
- <include>org.yaml:snakeyaml</include>
- <include>org.objenesis:*</include>
-
- <include>org.apache.avro:*</include>
- <!-- Avro transitive dependencies-->
- <include>org.codehaus.jackson:jackson-core-asl</include>
- <include>org.codehaus.jackson:jackson-mapper-asl</include>
- <include>com.thoughtworks.paranamer:paranamer</include>
- <include>org.apache.commons:commons-compress</include>
- <include>org.tukaani:xz</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>org.apache.pulsar:pulsar-client-original</artifact>
- <includes>
- <include>**</include>
- </includes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>org.apache.kafka.clients.producer.KafkaProducer</pattern>
- <shadedPattern>org.apache.kafka.clients.producer.OriginalKafkaProducer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.producer.PulsarKafkaProducer</pattern>
- <shadedPattern>org.apache.kafka.clients.producer.KafkaProducer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.KafkaConsumer</pattern>
- <shadedPattern>org.apache.kafka.clients.consumer.OriginalKafkaConsumer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarKafkaConsumer</pattern>
- <shadedPattern>org.apache.kafka.clients.consumer.KafkaConsumer</shadedPattern>
- </relocation>
-
- <!-- General relocation rules for Pulsar client dependencies -->
-
- <relocation>
- <pattern>org.asynchttpclient</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.datasketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.sketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.eclipse.jetty</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.reactivestreams</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.reactivestreams</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.typesafe</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.typesafe</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.memory</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.memory</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.objenesis</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.objenesis</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.yaml</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.yaml</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.avro</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.avro</shadedPattern>
- <excludes>
- <exclude>org.apache.avro.reflect.AvroAlias</exclude>
- <exclude>org.apache.avro.reflect.AvroDefault</exclude>
- <exclude>org.apache.avro.reflect.AvroEncode</exclude>
- <exclude>org.apache.avro.reflect.AvroIgnore</exclude>
- <exclude>org.apache.avro.reflect.AvroMeta</exclude>
- <exclude>org.apache.avro.reflect.AvroName</exclude>
- <exclude>org.apache.avro.reflect.AvroSchema</exclude>
- <exclude>org.apache.avro.reflect.Nullable</exclude>
- <exclude>org.apache.avro.reflect.Stringable</exclude>
- <exclude>org.apache.avro.reflect.Union</exclude>
- </excludes>
- </relocation>
- <!--Avro transitive dependencies-->
- <relocation>
- <pattern>org.codehaus.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.codehaus.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.thoughtworks.paranamer</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.thoughtworks.paranamer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.tukaani</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.tukaani</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.bookkeeper</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.bookkeeper</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <!-- This plugin is used to run a script after the package phase in order to rename
- libnetty_transport_native_epoll_x86_64.so from Netty into
- liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so
- to reflect the shade that is being applied.
- -->
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
- <executions>
- <execution>
- <id>rename-epoll-library</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${project.parent.basedir}/../src/${rename.netty.native.libs}</executable>
- <arguments>
- <argument>${project.artifactId}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_8/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_8/pom.xml
deleted file mode 100644
index de5bff1..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_8/pom.xml
+++ /dev/null
@@ -1,413 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka_0_8</artifactId>
- <name>Pulsar Kafka compatibility 0.8 :: API</name>
-
- <description>Drop-in replacement for Kafka client library that publishes and consumes
- messages on Pulsar topics</description>
-
- <properties>
- <kafka_0_8.version>0.8.1.1</kafka_0_8.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka_0_8-original</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <artifactSet>
- <includes>
- <include>org.apache.kafka:kafka_2.9.2</include>
- <include>org.apache.pulsar:pulsar-client-kafka_0_8-original</include>
- <include>org.apache.pulsar:pulsar-client-original</include>
- <include>org.apache.commons:commons-lang3</include>
- <include>commons-codec:commons-codec</include>
- <include>org.apache.bookkeeper:bookkeeper-common-allocator</include>
- <include>commons-collections:commons-collections</include>
- <include>org.asynchttpclient:*</include>
- <include>io.netty:netty-codec-http</include>
- <include>io.netty:netty-transport-native-epoll</include>
- <include>org.reactivestreams:reactive-streams</include>
- <include>com.typesafe.netty:netty-reactive-streams</include>
- <include>org.javassist:javassist</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>com.google.guava:guava</include>
- <include>com.google.code.gson:gson</include>
- <include>com.fasterxml.jackson.core</include>
- <include>com.fasterxml.jackson.module</include>
- <include>com.fasterxml.jackson.dataformat</include>
- <include>io.netty:netty</include>
- <include>io.netty:netty-*</include>
- <include>org.apache.pulsar:pulsar-common</include>
- <include>org.apache.bookkeeper:circe-checksum</include>
- <include>com.yahoo.datasketches:sketches-core</include>
- <include>org.eclipse.jetty:*</include>
- <include>com.yahoo.datasketches:*</include>
- <include>commons-*:*</include>
- <include>org.yaml:snakeyaml</include>
- <include>org.objenesis:*</include>
-
- <include>org.apache.avro:*</include>
- <!-- Avro transitive dependencies-->
- <include>org.codehaus.jackson:jackson-core-asl</include>
- <include>org.codehaus.jackson:jackson-mapper-asl</include>
- <include>com.thoughtworks.paranamer:paranamer</include>
- <include>org.apache.commons:commons-compress</include>
- <include>org.tukaani:xz</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>org.apache.pulsar:pulsar-client-original</artifact>
- <includes>
- <include>**</include>
- </includes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>kafka.javaapi.producer.Producer</pattern>
- <shadedPattern>kafka.javaapi.producer.OriginalProducer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.producer.PulsarKafkaProducer</pattern>
- <shadedPattern>kafka.javaapi.producer.Producer</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.consumer.ConsumerConnector</pattern>
- <shadedPattern>kafka.javaapi.consumer.OriginalConsumerConnector</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.ConsumerConnector</pattern>
- <shadedPattern>kafka.javaapi.consumer.ConsumerConnector</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.consumer.KafkaStream</pattern>
- <shadedPattern>kafka.consumer.OriginalKafkaStream</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarKafkaStream</pattern>
- <shadedPattern>kafka.consumer.KafkaStream</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.message.MessageAndMetadata</pattern>
- <shadedPattern>kafka.message.OriginalMessageAndMetadata</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarMessageAndMetadata</pattern>
- <shadedPattern>kafka.message.MessageAndMetadata</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.consumer.ConsumerIterator</pattern>
- <shadedPattern>kafka.consumer.OriginalConsumerIterator</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.ConsumerIterator</pattern>
- <shadedPattern>kafka.consumer.ConsumerIterator</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.consumer.Consumer</pattern>
- <shadedPattern>kafka.consumer.OriginalConsumer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarKafkaConsumer</pattern>
- <shadedPattern>kafka.consumer.Consumer</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarConsumerConfig</pattern>
- <shadedPattern>kafka.consumer.ConsumerConfig</shadedPattern>
- </relocation>
-
- <!-- low-level consumer -->
- <relocation>
- <pattern>kafka.javaapi.consumer.SimpleConsumer</pattern>
- <shadedPattern>kafka.javaapi.consumer.OriginalSimpleConsumer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarKafkaSimpleConsumer</pattern>
- <shadedPattern>kafka.javaapi.consumer.SimpleConsumer</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetFetchResponse</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetFetchResponse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetFetchResponse</pattern>
- <shadedPattern>kafka.javaapi.OffsetFetchResponse</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetFetchRequest</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetFetchRequest</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetFetchRequest</pattern>
- <shadedPattern>kafka.javaapi.OffsetFetchRequest</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetCommitRequest</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetCommitRequest</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetCommitRequest</pattern>
- <shadedPattern>kafka.javaapi.OffsetCommitRequest</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.FetchResponse</pattern>
- <shadedPattern>kafka.javaapi.OriginalFetchResponse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarFetchResponse</pattern>
- <shadedPattern>kafka.javaapi.FetchResponse</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.message.MessageAndOffset</pattern>
- <shadedPattern>kafka.message.MessageAndOffset</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.message.ByteBufferMessageSet</pattern>
- <shadedPattern>kafka.javaapi.message.OriginalByteBufferMessageSet</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarByteBufferMessageSet</pattern>
- <shadedPattern>kafka.javaapi.message.ByteBufferMessageSet</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.message.Message</pattern>
- <shadedPattern>kafka.message.OriginalMessage</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarMessage</pattern>
- <shadedPattern>kafka.message.Message</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.TopicMetadataResponse</pattern>
- <shadedPattern>kafka.javaapi.OriginalTopicMetadataResponse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarTopicMetadataResponse</pattern>
- <shadedPattern>kafka.javaapi.TopicMetadataResponse</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetCommitResponse</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetCommitResponse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetCommitResponse</pattern>
- <shadedPattern>kafka.javaapi.OffsetCommitResponse</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.common.OffsetMetadataAndError</pattern>
- <shadedPattern>kafka.common.OriginalOffsetMetadataAndError</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetMetadataAndError</pattern>
- <shadedPattern>kafka.common.OffsetMetadataAndError</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetResponse</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetResponse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetResponse</pattern>
- <shadedPattern>kafka.javaapi.OffsetResponse</shadedPattern>
- </relocation>
-
- <relocation>
- <pattern>kafka.javaapi.OffsetRequest</pattern>
- <shadedPattern>kafka.javaapi.OriginalOffsetRequest</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.simple.consumer.PulsarOffsetRequest</pattern>
- <shadedPattern>kafka.javaapi.OffsetRequest</shadedPattern>
- </relocation>
-
-
- <!-- General relocation rules for Pulsar client dependencies -->
-
- <relocation>
- <pattern>org.asynchttpclient</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.datasketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.sketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.eclipse.jetty</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.reactivestreams</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.reactivestreams</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.typesafe</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.typesafe</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.memory</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.memory</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.objenesis</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.objenesis</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.yaml</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.yaml</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.avro</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.avro</shadedPattern>
- <excludes>
- <exclude>org.apache.avro.reflect.AvroAlias</exclude>
- <exclude>org.apache.avro.reflect.AvroDefault</exclude>
- <exclude>org.apache.avro.reflect.AvroEncode</exclude>
- <exclude>org.apache.avro.reflect.AvroIgnore</exclude>
- <exclude>org.apache.avro.reflect.AvroMeta</exclude>
- <exclude>org.apache.avro.reflect.AvroName</exclude>
- <exclude>org.apache.avro.reflect.AvroSchema</exclude>
- <exclude>org.apache.avro.reflect.Nullable</exclude>
- <exclude>org.apache.avro.reflect.Stringable</exclude>
- <exclude>org.apache.avro.reflect.Union</exclude>
- </excludes>
- </relocation>
- <!--Avro transitive dependencies-->
- <relocation>
- <pattern>org.codehaus.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.codehaus.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.thoughtworks.paranamer</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.thoughtworks.paranamer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.tukaani</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.tukaani</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.bookkeeper</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.bookkeeper</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <!-- This plugin is used to run a script after the package phase in order to rename
- libnetty_transport_native_epoll_x86_64.so from Netty into
- liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so
- to reflect the shade that is being applied.
- -->
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
- <executions>
- <execution>
- <id>rename-epoll-library</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${project.parent.basedir}/../src/${rename.netty.native.libs}</executable>
- <arguments>
- <argument>${project.artifactId}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_9/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_9/pom.xml
deleted file mode 100644
index 4980544..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded_0_9/pom.xml
+++ /dev/null
@@ -1,251 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka_0_9</artifactId>
- <name>Pulsar Kafka compatibility 0.9 :: API</name>
-
- <description>Drop-in replacement for Kafka client library that publishes and consumes
- messages on Pulsar topics</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka_0_9-original</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
- <artifactSet>
- <includes>
- <include>org.apache.kafka:kafka-clients</include>
- <include>org.apache.pulsar:pulsar-client-kafka_0_9-original</include>
- <include>org.apache.pulsar:pulsar-client-original</include>
- <include>org.apache.commons:commons-lang3</include>
- <include>commons-codec:commons-codec</include>
- <include>org.apache.bookkeeper:bookkeeper-common-allocator</include>
- <include>commons-collections:commons-collections</include>
- <include>org.asynchttpclient:*</include>
- <include>io.netty:netty-codec-http</include>
- <include>io.netty:netty-transport-native-epoll</include>
- <include>org.reactivestreams:reactive-streams</include>
- <include>com.typesafe.netty:netty-reactive-streams</include>
- <include>org.javassist:javassist</include>
- <include>com.google.protobuf:protobuf-java</include>
- <include>com.google.guava:guava</include>
- <include>com.google.code.gson:gson</include>
- <include>com.fasterxml.jackson.core</include>
- <include>com.fasterxml.jackson.module</include>
- <include>com.fasterxml.jackson.dataformat</include>
- <include>io.netty:netty</include>
- <include>io.netty:netty-*</include>
- <include>org.apache.pulsar:pulsar-common</include>
- <include>org.apache.bookkeeper:circe-checksum</include>
- <include>com.yahoo.datasketches:sketches-core</include>
- <include>org.eclipse.jetty:*</include>
- <include>com.yahoo.datasketches:*</include>
- <include>commons-*:*</include>
- <include>org.yaml:snakeyaml</include>
- <include>org.objenesis:*</include>
-
- <include>org.apache.avro:*</include>
- <!-- Avro transitive dependencies-->
- <include>org.codehaus.jackson:jackson-core-asl</include>
- <include>org.codehaus.jackson:jackson-mapper-asl</include>
- <include>com.thoughtworks.paranamer:paranamer</include>
- <include>org.apache.commons:commons-compress</include>
- <include>org.tukaani:xz</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>org.apache.pulsar:pulsar-client-original</artifact>
- <includes>
- <include>**</include>
- </includes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>org.apache.kafka.clients.producer.KafkaProducer</pattern>
- <shadedPattern>org.apache.kafka.clients.producer.OriginalKafkaProducer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.producer.PulsarKafkaProducer</pattern>
- <shadedPattern>org.apache.kafka.clients.producer.KafkaProducer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.KafkaConsumer</pattern>
- <shadedPattern>org.apache.kafka.clients.consumer.OriginalKafkaConsumer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.kafka.clients.consumer.PulsarKafkaConsumer</pattern>
- <shadedPattern>org.apache.kafka.clients.consumer.KafkaConsumer</shadedPattern>
- </relocation>
-
- <!-- General relocation rules for Pulsar client dependencies -->
-
- <relocation>
- <pattern>org.asynchttpclient</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.datasketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.sketches</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.eclipse.jetty</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.eclipse</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.reactivestreams</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.reactivestreams</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.typesafe</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.typesafe</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.yahoo.memory</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.yahoo.memory</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.objenesis</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.objenesis</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.yaml</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.yaml</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.avro</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.avro</shadedPattern>
- <excludes>
- <exclude>org.apache.avro.reflect.AvroAlias</exclude>
- <exclude>org.apache.avro.reflect.AvroDefault</exclude>
- <exclude>org.apache.avro.reflect.AvroEncode</exclude>
- <exclude>org.apache.avro.reflect.AvroIgnore</exclude>
- <exclude>org.apache.avro.reflect.AvroMeta</exclude>
- <exclude>org.apache.avro.reflect.AvroName</exclude>
- <exclude>org.apache.avro.reflect.AvroSchema</exclude>
- <exclude>org.apache.avro.reflect.Nullable</exclude>
- <exclude>org.apache.avro.reflect.Stringable</exclude>
- <exclude>org.apache.avro.reflect.Union</exclude>
- </excludes>
- </relocation>
- <!--Avro transitive dependencies-->
- <relocation>
- <pattern>org.codehaus.jackson</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.codehaus.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.thoughtworks.paranamer</pattern>
- <shadedPattern>org.apache.pulsar.shade.com.thoughtworks.paranamer</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.tukaani</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.tukaani</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.bookkeeper</pattern>
- <shadedPattern>org.apache.pulsar.shade.org.apache.bookkeeper</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <!-- This plugin is used to run a script after the package phase in order to rename
- libnetty_transport_native_epoll_x86_64.so from Netty into
- liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so
- to reflect the shade that is being applied.
- -->
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
- <executions>
- <execution>
- <id>rename-epoll-library</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${project.parent.basedir}/../src/${rename.netty.native.libs}</executable>
- <arguments>
- <argument>${project.artifactId}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/pom.xml
deleted file mode 100644
index 3705ce4..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/pom.xml
+++ /dev/null
@@ -1,78 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka-tests</artifactId>
- <name>Pulsar Kafka compatibility :: Tests</name>
-
- <description>Tests to verify the correct shading configuration for the pulsar-client-kafka wrapper</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>bouncy-castle-bc</artifactId>
- <version>${project.version}</version>
- <classifier>pkg</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>testmocks</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java
deleted file mode 100644
index 3e39b8d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Bar;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Foo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Properties;
-
-public class ConsumerAvroExample {
-
- public static void main(String[] args) {
- String topic = "persistent://public/default/test-avro";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
- props.put("group.id", "my-subscription-name");
- props.put("enable.auto.commit", "false");
- props.put("key.deserializer", IntegerDeserializer.class.getName());
- props.put("value.deserializer", StringDeserializer.class.getName());
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
- @SuppressWarnings("resource")
- Consumer<Foo, Bar> consumer = new KafkaConsumer<>(props, fooSchema, barSchema);
- consumer.subscribe(Arrays.asList(topic));
-
- while (true) {
- ConsumerRecords<Foo, Bar> records = consumer.poll(100);
- records.forEach(record -> {
- log.info("Received record: {}", record);
- });
-
- // Commit last offset
- consumer.commitSync();
- }
- }
-
- private static final Logger log = LoggerFactory.getLogger(ConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java
deleted file mode 100644
index 983d7b7..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Arrays;
-import java.util.Properties;
-
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ConsumerExample {
-
- public static void main(String[] args) {
- String topic = "persistent://public/default/test";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
- props.put("group.id", "my-subscription-name");
- props.put("enable.auto.commit", "false");
- props.put("key.deserializer", IntegerDeserializer.class.getName());
- props.put("value.deserializer", StringDeserializer.class.getName());
-
- @SuppressWarnings("resource")
- Consumer<Integer, String> consumer = new KafkaConsumer<>(props);
- consumer.subscribe(Arrays.asList(topic));
-
- while (true) {
- ConsumerRecords<Integer, String> records = consumer.poll(100);
- records.forEach(record -> {
- log.info("Received record: {}", record);
- });
-
- // Commit last offset
- consumer.commitSync();
- }
- }
-
- private static final Logger log = LoggerFactory.getLogger(ConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java
deleted file mode 100644
index 5d9fcfc..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Bar;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Foo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Properties;
-
-public class ProducerAvroExample {
- public static void main(String[] args) {
- String topic = "persistent://public/default/test-avro";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
-
- props.put("key.serializer", IntegerSerializer.class.getName());
- props.put("value.serializer", StringSerializer.class.getName());
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
-
- Producer<Foo, Bar> producer = new KafkaProducer<>(props, fooSchema, barSchema);
-
- for (int i = 0; i < 10; i++) {
- producer.send(new ProducerRecord<Foo, Bar>(topic, i, foo, bar));
- log.info("Message {} sent successfully", i);
- }
-
- producer.flush();
- producer.close();
- }
-
- private static final Logger log = LoggerFactory.getLogger(ProducerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
deleted file mode 100644
index 34e008e..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Properties;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ProducerExample {
- public static void main(String[] args) {
- String topic = "persistent://public/default/test";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
-
- props.put("key.serializer", IntegerSerializer.class.getName());
- props.put("value.serializer", StringSerializer.class.getName());
-
- Producer<Integer, String> producer = new KafkaProducer<>(props);
-
- for (int i = 0; i < 10; i++) {
- producer.send(new ProducerRecord<Integer, String>(topic, i, Integer.toString(i)));
- log.info("Message {} sent successfully", i);
- }
-
- producer.flush();
- producer.close();
- }
-
- private static final Logger log = LoggerFactory.getLogger(ProducerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java
deleted file mode 100644
index 8120900..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples.utils;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-@Data
-@ToString
-@EqualsAndHashCode
-public class Bar {
- private boolean field1;
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java
deleted file mode 100644
index d584f51..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples.utils;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-import org.apache.avro.reflect.Nullable;
-
-@Data
-@ToString
-@EqualsAndHashCode
-public class Foo {
- @Nullable
- private String field1;
- @Nullable
- private String field2;
- private int field3;
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/pom.xml
deleted file mode 100644
index d485872..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/pom.xml
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka-tests_0_8</artifactId>
- <name>Pulsar Kafka-0.8 compatibility :: Tests</name>
-
- <description>Tests to verify the correct shading configuration for the pulsar-client-kafka wrapper</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka_0_8</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>bouncy-castle-bc</artifactId>
- <version>${project.version}</version>
- <classifier>pkg</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>testmocks</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/HighLevelConsumerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/HighLevelConsumerExample.java
deleted file mode 100644
index b00e19d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/HighLevelConsumerExample.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.pulsar.client.kafka.compat.examples.utils.Tweet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-
-import kafka.consumer.Consumer;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-import kafka.message.MessageAndMetadata;
-import kafka.serializer.StringDecoder;
-
-public class HighLevelConsumerExample {
-
- static class Arguments {
-
- @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
- boolean help;
-
- @Parameter(names = { "-u", "--service-url" }, description = "Service url", required = false)
- public String serviceUrl = "pulsar://localhost:6650";
-
- @Parameter(names = { "-t", "--topic-name" }, description = "Topic name", required = false)
- public String topicName = "persistent://public/default/test";
-
- @Parameter(names = { "-g", "--group-name" }, description = "Group name", required = false)
- public String groupName = "high-level";
-
- @Parameter(names = { "-m", "--total-messages" }, description = "total number message to publish")
- public int totalMessages = 1;
-
- @Parameter(names = { "-a", "--auto-commit-disable" }, description = "auto commit disable")
- public boolean autoCommitDisable;
- }
-
- public static void main(String[] args) {
-
- final Arguments arguments = new Arguments();
- JCommander jc = new JCommander(arguments);
- jc.setProgramName("pulsar-kafka-test");
-
- try {
- jc.parse(args);
- } catch (ParameterException e) {
- System.out.println(e.getMessage());
- jc.usage();
- System.exit(-1);
- }
-
- if (arguments.help) {
- jc.usage();
- System.exit(-1);
- }
-
- consumeMessage(arguments);
- }
-
- private static void consumeMessage(Arguments arguments) {
-
- Properties properties = new Properties();
- properties.put("zookeeper.connect", arguments.serviceUrl);
- properties.put("group.id", arguments.groupName);
- properties.put("consumer.id", "cons1");
- properties.put("auto.commit.enable", Boolean.toString(!arguments.autoCommitDisable));
- properties.put("auto.commit.interval.ms", "100");
- properties.put("queued.max.message.chunks", "100");
-
- ConsumerConfig conSConfig = new ConsumerConfig(properties);
- ConsumerConnector connector = Consumer.createJavaConsumerConnector(conSConfig);
- Map<String, Integer> topicCountMap = Collections.singletonMap(arguments.topicName, 2);
- Map<String, List<KafkaStream<String, Tweet>>> streams = connector.createMessageStreams(topicCountMap,
- new StringDecoder(null), new Tweet.TestDecoder());
-
- int count = 0;
- while (count < arguments.totalMessages || arguments.totalMessages == -1) {
- for (int i = 0; i < streams.size(); i++) {
- List<KafkaStream<String, Tweet>> kafkaStreams = streams.get(arguments.topicName);
- for (KafkaStream<String, Tweet> kafkaStream : kafkaStreams) {
- for (MessageAndMetadata<String, Tweet> record : kafkaStream) {
- log.info("Received tweet: {}-{}", record.message().userName, record.message().message);
- count++;
- }
- }
- }
- }
-
- connector.shutdown();
-
- log.info("successfully consumed message {}", count);
- }
-
- private static final Logger log = LoggerFactory.getLogger(HighLevelConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/LowLevelConsumerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/LowLevelConsumerExample.java
deleted file mode 100644
index d77617d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/LowLevelConsumerExample.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.kafka.clients.simple.consumer.PulsarMsgAndOffset;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Tweet;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Tweet.TestDecoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-
-import kafka.api.FetchRequestBuilder;
-import kafka.common.OffsetMetadataAndError;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.OffsetCommitRequest;
-import kafka.javaapi.consumer.SimpleConsumer;
-import kafka.message.MessageAndOffset;
-
-public class LowLevelConsumerExample {
-
- static class Arguments {
- @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
- boolean help;
-
- @Parameter(names = { "-u", "--service-url" }, description = "Service url", required = false)
- public String serviceUrl = "pulsar://localhost:6650";
-
- @Parameter(names = { "-hu",
- "--http-service-url" }, description = "Http ervice url to make admin api call", required = false)
- public String httpServiceUrl = "pulsar://localhost:8080";
-
- @Parameter(names = { "-t", "--topic-name" }, description = "Topic name", required = false)
- public String topicName = "persistent://public/default/test";
-
- @Parameter(names = { "-g", "--group-name" }, description = "Group name", required = false)
- public String groupName = "low-level";
-
- @Parameter(names = { "-m", "--total-messages" }, description = "total number message to publish")
- public int totalMessages = 1;
-
- @Parameter(names = { "-p",
- "--partition-index" }, description = "Partition-index (-1 if topic is not partitioned)", required = false)
- public int partitionIndex = -1;
- }
-
- public static void main(String[] args) {
-
- final Arguments arguments = new Arguments();
- JCommander jc = new JCommander(arguments);
- jc.setProgramName("pulsar-kafka-test");
-
- try {
- jc.parse(args);
- } catch (ParameterException e) {
- System.out.println(e.getMessage());
- jc.usage();
- System.exit(-1);
- }
-
- if (arguments.help) {
- jc.usage();
- System.exit(-1);
- }
-
- try {
- consumeMessage(arguments);
- } catch (Exception e) {
- log.error("Failed to consume message", e);
- }
- }
-
- private static void consumeMessage(Arguments arguments) {
-
- Properties properties = new Properties();
- properties.put(SimpleConsumer.HTTP_SERVICE_URL, arguments.httpServiceUrl);
- SimpleConsumer consumer = new SimpleConsumer(arguments.serviceUrl, 0, 0, 0, "clientId", properties);
-
- long readOffset = kafka.api.OffsetRequest.EarliestTime();
- kafka.api.FetchRequest fReq = new FetchRequestBuilder().clientId("c1")
- .addFetch(arguments.topicName, arguments.partitionIndex, readOffset, 100000).build();
- FetchResponse fetchResponse = consumer.fetch(fReq);
-
- TestDecoder decoder = new TestDecoder();
- int count = 0;
- while (count < arguments.totalMessages || arguments.totalMessages == -1) {
- // 1. Read from topic without subscription/consumer-group name.
- for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(arguments.topicName,
- arguments.partitionIndex)) {
- MessageId msgIdOffset = (messageAndOffset instanceof PulsarMsgAndOffset)
- ? ((PulsarMsgAndOffset) messageAndOffset).getFullOffset()
- : null;
- long currentOffset = messageAndOffset.offset();
- if (currentOffset < readOffset) {
- continue;
- }
-
- ByteBuffer payload = messageAndOffset.message().payload();
-
- byte[] bytes = new byte[payload.limit()];
- payload.get(bytes);
- Tweet tweet = decoder.fromBytes(bytes);
- log.info("Received tweet: {}-{}", tweet.userName, tweet.message);
- count++;
-
- TopicAndPartition topicPartition = new TopicAndPartition(arguments.topicName, arguments.partitionIndex);
- OffsetMetadataAndError offsetError = new OffsetMetadataAndError(msgIdOffset, null, (short) 0);
- Map<TopicAndPartition, OffsetMetadataAndError> requestInfo = Collections.singletonMap(topicPartition,
- offsetError);
- // 2. Commit offset for a given topic and subscription-name/consumer-name.
- OffsetCommitRequest offsetReq = new OffsetCommitRequest(arguments.groupName, requestInfo, (short) -1, 0,
- "c1");
- consumer.commitOffsets(offsetReq);
- }
- }
-
- consumer.close();
- }
-
- private static final Logger log = LoggerFactory.getLogger(LowLevelConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
deleted file mode 100644
index 360f79d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Properties;
-
-import org.apache.pulsar.client.kafka.compat.examples.utils.Tweet;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Tweet.TestEncoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.Partitioner;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.StringEncoder;
-
-public class ProducerExample {
-
- public static final String BROKER_URL = "metadata.broker.list";
- public static final String PRODUCER_TYPE = "producer.type";
- public static final String SERIALIZER_CLASS = "serializer.class";
- public static final String KEY_SERIALIZER_CLASS = "key.serializer.class";
- public static final String PARTITIONER_CLASS = "partitioner.class";
- public static final String COMPRESSION_CODEC = "compression.codec";
- public static final String QUEUE_BUFFERING_MAX_MS = "queue.buffering.max.ms";
- public static final String QUEUE_BUFFERING_MAX_MESSAGES = "queue.buffering.max.messages";
- public static final String QUEUE_ENQUEUE_TIMEOUT_MS = "queue.enqueue.timeout.ms";
- public static final String BATCH_NUM_MESSAGES = "batch.num.messages";
- public static final String CLIENT_ID = "client.id";
-
- static class Arguments {
-
- @Parameter(names = { "-u", "--service-url" }, description = "Service url", required = false)
- public String serviceUrl = "pulsar://localhost:6650";
-
- @Parameter(names = { "-t", "--topic-name" }, description = "Topic name", required = false)
- public String topicName = "persistent://public/default/test";
-
- @Parameter(names = { "-m", "--total-messages" }, description = "total number message to publish")
- public int totalMessages = 1;
-
- @Parameter(names = { "-mn", "--message-name" }, description = "Message payload value", required = false)
- public String messageValue = "Hello-world";
-
- @Parameter(names = { "-h", "--help" }, description = "Help message", help = true)
- boolean help;
- }
-
- public static void main(String[] args) {
-
- final Arguments arguments = new Arguments();
- JCommander jc = new JCommander(arguments);
- jc.setProgramName("pulsar-kafka-test");
-
- try {
- jc.parse(args);
- } catch (ParameterException e) {
- System.out.println(e.getMessage());
- jc.usage();
- System.exit(-1);
- }
-
- if (arguments.help) {
- jc.usage();
- System.exit(-1);
- }
-
- publishMessage(arguments);
- }
-
- private static void publishMessage(Arguments arguments) {
- // (2) Create producer
- Properties properties2 = new Properties();
- properties2.put(BROKER_URL, arguments.serviceUrl);
- properties2.put(PRODUCER_TYPE, "sync");
- properties2.put(SERIALIZER_CLASS, TestEncoder.class.getName());
- properties2.put(KEY_SERIALIZER_CLASS, StringEncoder.class.getName());
- properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
- properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
- properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
- properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
- properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
- properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
- properties2.put(CLIENT_ID, "test");
- ProducerConfig config = new ProducerConfig(properties2);
- Producer<String, Tweet> producer = new Producer<>(config);
-
- String name = "user";
- String msg = arguments.messageValue;
- for (int i = 0; i < arguments.totalMessages; i++) {
- String sendMessage = msg + i;
- Tweet tweet = new Tweet(name, sendMessage);
- KeyedMessage<String, Tweet> message = new KeyedMessage<>(arguments.topicName, name, tweet);
- producer.send(message);
- }
-
- producer.close();
- log.info("Successfully published messages {}", arguments.totalMessages);
-
- }
-
- public static class TestPartitioner implements Partitioner {
- @Override
- public int partition(Object obj, int totalPartition) {
- return obj.hashCode() % totalPartition;
- }
- }
-
- private static final Logger log = LoggerFactory.getLogger(ProducerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Tweet.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Tweet.java
deleted file mode 100644
index d23411e..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_8/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Tweet.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples.utils;
-
-import java.io.Serializable;
-
-import com.google.common.base.Objects;
-
-import kafka.serializer.Decoder;
-import kafka.serializer.Encoder;
-
-public class Tweet implements Serializable {
- private static final long serialVersionUID = 1L;
- public String userName;
- public String message;
-
- public Tweet(String userName, String message) {
- super();
- this.userName = userName;
- this.message = message;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(userName, message);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof Tweet) {
- Tweet tweet = (Tweet) obj;
- return Objects.equal(this.userName, tweet.userName) && Objects.equal(this.message, tweet.message);
- }
- return false;
- }
-
- public static class TestEncoder implements Encoder<Tweet> {
- @Override
- public byte[] toBytes(Tweet tweet) {
- return (tweet.userName + "," + tweet.message).getBytes();
- }
- }
-
- public static class TestDecoder implements Decoder<Tweet> {
- @Override
- public Tweet fromBytes(byte[] input) {
- String[] tokens = (new String(input)).split(",");
- return new Tweet(tokens[0], tokens[1]);
- }
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/pom.xml
deleted file mode 100644
index 9c0df85..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/pom.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka_0_9-tests</artifactId>
- <name>Pulsar Kafka compatibility 0.9 :: Tests</name>
-
- <description>Tests to verify the correct shading configuration for the pulsar-client-kafka wrapper</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-kafka_0_9</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>bouncy-castle-bc</artifactId>
- <version>${project.version}</version>
- <classifier>pkg</classifier>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>testmocks</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java
deleted file mode 100644
index 3e39b8d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerAvroExample.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Bar;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Foo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Properties;
-
-public class ConsumerAvroExample {
-
- public static void main(String[] args) {
- String topic = "persistent://public/default/test-avro";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
- props.put("group.id", "my-subscription-name");
- props.put("enable.auto.commit", "false");
- props.put("key.deserializer", IntegerDeserializer.class.getName());
- props.put("value.deserializer", StringDeserializer.class.getName());
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
- @SuppressWarnings("resource")
- Consumer<Foo, Bar> consumer = new KafkaConsumer<>(props, fooSchema, barSchema);
- consumer.subscribe(Arrays.asList(topic));
-
- while (true) {
- ConsumerRecords<Foo, Bar> records = consumer.poll(100);
- records.forEach(record -> {
- log.info("Received record: {}", record);
- });
-
- // Commit last offset
- consumer.commitSync();
- }
- }
-
- private static final Logger log = LoggerFactory.getLogger(ConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java
deleted file mode 100644
index 983d7b7..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ConsumerExample.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Arrays;
-import java.util.Properties;
-
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ConsumerExample {
-
- public static void main(String[] args) {
- String topic = "persistent://public/default/test";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
- props.put("group.id", "my-subscription-name");
- props.put("enable.auto.commit", "false");
- props.put("key.deserializer", IntegerDeserializer.class.getName());
- props.put("value.deserializer", StringDeserializer.class.getName());
-
- @SuppressWarnings("resource")
- Consumer<Integer, String> consumer = new KafkaConsumer<>(props);
- consumer.subscribe(Arrays.asList(topic));
-
- while (true) {
- ConsumerRecords<Integer, String> records = consumer.poll(100);
- records.forEach(record -> {
- log.info("Received record: {}", record);
- });
-
- // Commit last offset
- consumer.commitSync();
- }
- }
-
- private static final Logger log = LoggerFactory.getLogger(ConsumerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java
deleted file mode 100644
index 5d9fcfc..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerAvroExample.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Bar;
-import org.apache.pulsar.client.kafka.compat.examples.utils.Foo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Properties;
-
-public class ProducerAvroExample {
- public static void main(String[] args) {
- String topic = "persistent://public/default/test-avro";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
-
- props.put("key.serializer", IntegerSerializer.class.getName());
- props.put("value.serializer", StringSerializer.class.getName());
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
-
- Producer<Foo, Bar> producer = new KafkaProducer<>(props, fooSchema, barSchema);
-
- for (int i = 0; i < 10; i++) {
- producer.send(new ProducerRecord<Foo, Bar>(topic, i, foo, bar));
- log.info("Message {} sent successfully", i);
- }
-
- producer.flush();
- producer.close();
- }
-
- private static final Logger log = LoggerFactory.getLogger(ProducerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
deleted file mode 100644
index f089b26..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/ProducerExample.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples;
-
-import java.util.Properties;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ProducerExample {
- public static void main(String[] args) {
- String topic = "persistent://public/default/test";
-
- Properties props = new Properties();
- props.put("bootstrap.servers", "pulsar://localhost:6650");
- props.put("key.serializer", IntegerSerializer.class.getName());
- props.put("value.serializer", StringSerializer.class.getName());
-
- Producer<Integer, String> producer = new KafkaProducer<>(props);
-
- for (int i = 0; i < 10; i++) {
- producer.send(new ProducerRecord<Integer, String>(topic, i, Integer.toString(i)));
- log.info("Message {} sent successfully", i);
- }
-
- producer.flush();
- producer.close();
- }
-
- private static final Logger log = LoggerFactory.getLogger(ProducerExample.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java
deleted file mode 100644
index 8120900..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Bar.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples.utils;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-@Data
-@ToString
-@EqualsAndHashCode
-public class Bar {
- private boolean field1;
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java b/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java
deleted file mode 100644
index d584f51..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka-tests_0_9/src/test/java/org/apache/pulsar/client/kafka/compat/examples/utils/Foo.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat.examples.utils;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-import org.apache.avro.reflect.Nullable;
-
-@Data
-@ToString
-@EqualsAndHashCode
-public class Foo {
- @Nullable
- private String field1;
- @Nullable
- private String field2;
- private int field3;
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka/pom.xml
deleted file mode 100644
index c666169..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/pom.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-client-kafka-original</artifactId>
- <name>Pulsar Kafka compatibility :: API (original)</name>
-
- <description>Kafka client library that publishes and consumes messages on Pulsar topics</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-original</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka-client.version}</version>
- <exclusions>
- <exclusion>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
deleted file mode 100644
index 06b1c6e..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
+++ /dev/null
@@ -1,743 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.pulsar.client.api.MessageListener;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.SubscriptionInitialPosition;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarConsumerKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarKafkaSchema;
-import org.apache.pulsar.client.util.ConsumerName;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
-
-@Slf4j
-public class PulsarKafkaConsumer<K, V> implements Consumer<K, V>, MessageListener<byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- private final PulsarClient client;
-
- private final Schema<K> keySchema;
- private final Schema<V> valueSchema;
-
- private final String groupId;
- private final boolean isAutoCommit;
-
- private final ConcurrentMap<TopicPartition, org.apache.pulsar.client.api.Consumer<byte[]>> consumers = new ConcurrentHashMap<>();
-
- private final Map<TopicPartition, Long> lastReceivedOffset = new ConcurrentHashMap<>();
- private final Map<TopicPartition, OffsetAndMetadata> lastCommittedOffset = new ConcurrentHashMap<>();
- private final Set<TopicPartition> unpolledPartitions = new HashSet<>();
- private final SubscriptionInitialPosition strategy;
-
- private List<ConsumerInterceptor<K, V>> interceptors;
-
- private volatile boolean closed = false;
-
- private final int maxRecordsInSinglePoll;
-
- private final Properties properties;
-
- private static class QueueItem {
- final org.apache.pulsar.client.api.Consumer<byte[]> consumer;
- final Message<byte[]> message;
-
- QueueItem(org.apache.pulsar.client.api.Consumer<byte[]> consumer, Message<byte[]> message) {
- this.consumer = consumer;
- this.message = message;
- }
- }
-
- // Since a single Kafka consumer can receive from multiple topics, we need to multiplex all the different
- // topics/partitions into a single queues
- private final BlockingQueue<QueueItem> receivedMessages = new ArrayBlockingQueue<>(1000);
-
- public PulsarKafkaConsumer(Map<String, Object> configs) {
- this(new ConsumerConfig(configs), null, null);
- }
-
- public PulsarKafkaConsumer(Map<String, Object> configs, Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer) {
- this(new ConsumerConfig(configs),
- new PulsarKafkaSchema<K>(keyDeserializer), new PulsarKafkaSchema<V>(valueDeserializer));
- }
-
- public PulsarKafkaConsumer(Map<String, Object> configs, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ConsumerConfig(configs), keySchema, valueSchema);
- }
-
- public PulsarKafkaConsumer(Properties properties) {
- this(new ConsumerConfig(properties), null, null);
- }
-
- public PulsarKafkaConsumer(Properties properties, Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer) {
- this(new ConsumerConfig(properties),
- new PulsarKafkaSchema<>(keyDeserializer), new PulsarKafkaSchema<>(valueDeserializer));
- }
-
- public PulsarKafkaConsumer(Properties properties, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ConsumerConfig(properties), keySchema, valueSchema);
- }
-
- @SuppressWarnings("unchecked")
- private PulsarKafkaConsumer(ConsumerConfig consumerConfig, Schema<K> keySchema, Schema<V> valueSchema) {
-
- if (keySchema == null) {
- Deserializer<K> kafkaKeyDeserializer = consumerConfig.getConfiguredInstance(
- ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Deserializer.class);
- kafkaKeyDeserializer.configure(consumerConfig.originals(), true);
- this.keySchema = new PulsarKafkaSchema<>(kafkaKeyDeserializer);
- } else {
- this.keySchema = keySchema;
- consumerConfig.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG);
- }
-
- if (valueSchema == null) {
- Deserializer<V> kafkaValueDeserializer = consumerConfig.getConfiguredInstance(
- ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class);
- kafkaValueDeserializer.configure(consumerConfig.originals(), true);
- this.valueSchema = new PulsarKafkaSchema<>(kafkaValueDeserializer);
- } else {
- this.valueSchema = valueSchema;
- consumerConfig.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG);
- }
-
- groupId = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG);
- isAutoCommit = consumerConfig.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
- strategy = getStrategy(consumerConfig.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
- log.info("Offset reset strategy has been assigned value {}", strategy);
-
- String serviceUrl = consumerConfig.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG).get(0);
-
- // If MAX_POLL_RECORDS_CONFIG is provided then use the config, else use default value.
- if(consumerConfig.values().containsKey(ConsumerConfig.MAX_POLL_RECORDS_CONFIG)){
- maxRecordsInSinglePoll = consumerConfig.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG);
- } else {
- maxRecordsInSinglePoll = 1000;
- }
-
- interceptors = (List) consumerConfig.getConfiguredInstances(
- ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptor.class);
-
- this.properties = new Properties();
- consumerConfig.originals().forEach((k, v) -> properties.put(k, v));
- ClientBuilder clientBuilder = PulsarClientKafkaConfig.getClientBuilder(properties);
- // Since this client instance is going to be used just for the consumers, we can enable Nagle to group
- // all the acknowledgments sent to broker within a short time frame
- clientBuilder.enableTcpNoDelay(false);
- try {
- client = clientBuilder.serviceUrl(serviceUrl).build();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- private SubscriptionInitialPosition getStrategy(final String strategy) {
- switch(strategy) {
- case "earliest":
- return SubscriptionInitialPosition.Earliest;
- default:
- return SubscriptionInitialPosition.Latest;
- }
- }
-
- @Override
- public void received(org.apache.pulsar.client.api.Consumer<byte[]> consumer, Message<byte[]> msg) {
- // Block listener thread if the application is slowing down
- try {
- receivedMessages.put(new QueueItem(consumer, msg));
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- if (closed) {
- // Consumer was closed and the thread was interrupted. Nothing to worry about here
- } else {
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public Set<TopicPartition> assignment() {
- throw new UnsupportedOperationException("Cannot access the partitions assignements");
- }
-
- /**
- * Get the current subscription. Will return the same topics used in the most recent call to
- * {@link #subscribe(Collection, ConsumerRebalanceListener)}, or an empty set if no such call has been made.
- *
- * @return The set of topics currently subscribed to
- */
- @Override
- public Set<String> subscription() {
- return consumers.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
- }
-
- @Override
- public void subscribe(Collection<String> topics) {
- subscribe(topics, null);
- }
-
- @Override
- public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
- List<CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>>> futures = new ArrayList<>();
-
- List<TopicPartition> topicPartitions = new ArrayList<>();
- try {
- for (String topic : topics) {
- // Create individual subscription on each partition, that way we can keep using the
- // acknowledgeCumulative()
- int numberOfPartitions = ((PulsarClientImpl) client).getNumberOfPartitions(topic).get();
-
- ConsumerBuilder<byte[]> consumerBuilder = PulsarConsumerKafkaConfig.getConsumerBuilder(client, properties);
- consumerBuilder.subscriptionType(SubscriptionType.Failover);
- consumerBuilder.messageListener(this);
- consumerBuilder.subscriptionName(groupId);
- if (numberOfPartitions > 1) {
- // Subscribe to each partition
- consumerBuilder.consumerName(ConsumerName.generateRandomName());
- for (int i = 0; i < numberOfPartitions; i++) {
- String partitionName = TopicName.get(topic).getPartition(i).toString();
- CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.clone()
- .topic(partitionName).subscribeAsync();
- int partitionIndex = i;
- TopicPartition tp = new TopicPartition(
- TopicName.get(topic).getPartitionedTopicName(),
- partitionIndex);
- futures.add(future.thenApply(consumer -> {
- log.info("Add consumer {} for partition {}", consumer, tp);
- consumers.putIfAbsent(tp, consumer);
- return consumer;
- }));
- topicPartitions.add(tp);
- }
- } else {
- // Topic has a single partition
- CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.topic(topic)
- .subscribeAsync();
- TopicPartition tp = new TopicPartition(
- TopicName.get(topic).getPartitionedTopicName(),
- 0);
- futures.add(future.thenApply(consumer -> {
- log.info("Add consumer {} for partition {}", consumer, tp);
- consumers.putIfAbsent(tp, consumer);
- return consumer;
- }));
- topicPartitions.add(tp);
- }
- }
- unpolledPartitions.addAll(topicPartitions);
-
- // Wait for all consumers to be ready
- futures.forEach(CompletableFuture::join);
-
- // Notify the listener is now owning all topics/partitions
- if (callback != null) {
- callback.onPartitionsAssigned(topicPartitions);
- }
-
- } catch (Exception e) {
- // Close all consumer that might have been successfully created
- futures.forEach(f -> {
- try {
- f.get().close();
- } catch (Exception e1) {
- // Ignore. Consumer already had failed
- }
- });
-
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void assign(Collection<TopicPartition> partitions) {
- throw new UnsupportedOperationException("Cannot manually assign partitions");
- }
-
- @Override
- public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
- throw new UnsupportedOperationException("Cannot subscribe with topic name pattern");
- }
-
- @Override
- public void subscribe(Pattern pattern) {
- throw new UnsupportedOperationException("Cannot subscribe with topic name pattern");
- }
-
- @Override
- public void unsubscribe() {
- consumers.values().forEach(c -> {
- try {
- c.unsubscribe();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- });
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ConsumerRecords<K, V> poll(long timeoutMillis) {
- try {
- QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
- if (item == null) {
- return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
- }
-
- Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();
-
- int numberOfRecords = 0;
-
- while (item != null) {
- TopicName topicName = TopicName.get(item.consumer.getTopic());
- String topic = topicName.getPartitionedTopicName();
- int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
- Message<byte[]> msg = item.message;
- MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
- long offset = MessageIdUtils.getOffset(msgId);
-
- TopicPartition tp = new TopicPartition(topic, partition);
- if (lastReceivedOffset.get(tp) == null && !unpolledPartitions.contains(tp)) {
- log.info("When polling offsets, invalid offsets were detected. Resetting topic partition {}", tp);
- resetOffsets(tp);
- }
-
- K key = getKey(topic, msg);
- if (valueSchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<V>) valueSchema).setTopic(topic);
- }
- V value = valueSchema.decode(msg.getData());
-
- TimestampType timestampType = TimestampType.LOG_APPEND_TIME;
- long timestamp = msg.getPublishTime();
-
- if (msg.getEventTime() > 0) {
- // If we have Event time, use that in preference
- timestamp = msg.getEventTime();
- timestampType = TimestampType.CREATE_TIME;
- }
-
- ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, timestamp,
- timestampType, -1, msg.hasKey() ? msg.getKey().length() : 0, msg.getData().length, key, value);
-
- records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);
-
- // Update last offset seen by application
- lastReceivedOffset.put(tp, offset);
- unpolledPartitions.remove(tp);
-
- if (++numberOfRecords >= maxRecordsInSinglePoll) {
- break;
- }
-
- // Check if we have an item already available
- item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
- }
-
- if (isAutoCommit && !records.isEmpty()) {
- // Commit the offset of previously dequeued messages
- commitAsync();
- }
-
- // If no interceptor is provided, interceptors list will an empty list, original ConsumerRecords will be return.
- return applyConsumerInterceptorsOnConsume(interceptors, new ConsumerRecords<>(records));
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public ConsumerRecords<K, V> poll(Duration duration) {
- return poll(duration.toMillis());
- }
-
- @SuppressWarnings("unchecked")
- private K getKey(String topic, Message<byte[]> msg) {
- if (!msg.hasKey()) {
- return null;
- }
-
- if (keySchema instanceof PulsarKafkaSchema) {
- PulsarKafkaSchema<K> pulsarKafkaSchema = (PulsarKafkaSchema) keySchema;
- Deserializer<K> kafkaDeserializer = pulsarKafkaSchema.getKafkaDeserializer();
- if (kafkaDeserializer instanceof StringDeserializer) {
- return (K) msg.getKey();
- }
- pulsarKafkaSchema.setTopic(topic);
- }
- // Assume base64 encoding
- byte[] data = Base64.getDecoder().decode(msg.getKey());
- return keySchema.decode(data);
-
- }
-
- @Override
- public void commitSync() {
- try {
- doCommitOffsets(getCurrentOffsetsMap()).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void commitSync(Duration duration) {
- commitSync();
- }
-
- @Override
- public void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) {
- try {
- doCommitOffsets(offsets).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void commitSync(Map<TopicPartition, OffsetAndMetadata> map, Duration duration) {
- commitSync(map);
- }
-
- @Override
- public void commitAsync() {
- doCommitOffsets(getCurrentOffsetsMap());
- }
-
- @Override
- public void commitAsync(OffsetCommitCallback callback) {
- Map<TopicPartition, OffsetAndMetadata> offsets = getCurrentOffsetsMap();
- doCommitOffsets(offsets).handle((v, throwable) -> {
- callback.onComplete(offsets, throwable != null ? new Exception(throwable) : null);
- return null;
- });
- }
-
- @Override
- public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
- doCommitOffsets(offsets).handle((v, throwable) -> {
- callback.onComplete(offsets, throwable != null ? new Exception(throwable) : null);
- return null;
- });
- }
-
- private CompletableFuture<Void> doCommitOffsets(Map<TopicPartition, OffsetAndMetadata> offsets) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- applyConsumerInterceptorsOnCommit(interceptors, offsets);
- offsets.forEach((topicPartition, offsetAndMetadata) -> {
- org.apache.pulsar.client.api.Consumer<byte[]> consumer = consumers.get(topicPartition);
- lastCommittedOffset.put(topicPartition, offsetAndMetadata);
- futures.add(consumer.acknowledgeCumulativeAsync(MessageIdUtils.getMessageId(offsetAndMetadata.offset())));
- });
-
- return FutureUtil.waitForAll(futures);
- }
-
- private Map<TopicPartition, OffsetAndMetadata> getCurrentOffsetsMap() {
- Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
- lastReceivedOffset.forEach((topicPartition, offset) -> {
- OffsetAndMetadata om = new OffsetAndMetadata(offset);
- offsets.put(topicPartition, om);
- });
-
- return offsets;
- }
-
- /**
- * Apply all onConsume methods in a list of ConsumerInterceptors.
- * Catch any exception during the process.
- *
- * @param interceptors Interceptors provided.
- * @param consumerRecords ConsumerRecords returned by calling {@link this#poll(long)}.
- * @return ConsumerRecords after applying all ConsumerInterceptor in interceptors list.
- */
- private ConsumerRecords applyConsumerInterceptorsOnConsume(List<ConsumerInterceptor<K, V>> interceptors, ConsumerRecords consumerRecords) {
- ConsumerRecords processedConsumerRecords = consumerRecords;
- for (ConsumerInterceptor interceptor : interceptors) {
- try {
- processedConsumerRecords = interceptor.onConsume(processedConsumerRecords);
- } catch (Exception e) {
- log.warn("Error executing onConsume for interceptor {}.", interceptor.getClass().getCanonicalName(), e);
- }
- }
- return processedConsumerRecords;
- }
-
- /**
- * Apply all onCommit methods in a list of ConsumerInterceptors.
- * Catch any exception during the process.
- *
- * @param interceptors Interceptors provided.
- * @param offsets Offsets need to be commit.
- */
- private void applyConsumerInterceptorsOnCommit(List<ConsumerInterceptor<K, V>> interceptors, Map<TopicPartition, OffsetAndMetadata> offsets) {
- for (ConsumerInterceptor interceptor : interceptors) {
- try {
- interceptor.onCommit(offsets);
- } catch (Exception e) {
- log.warn("Error executing onCommit for interceptor {}.", interceptor.getClass().getCanonicalName(), e);
- }
- }
- }
-
- @Override
- public void seek(TopicPartition partition, long offset) {
- MessageId msgId = MessageIdUtils.getMessageId(offset);
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(partition);
- if (c == null) {
- throw new IllegalArgumentException("Cannot seek on a partition where we are not subscribed");
- }
-
- try {
- c.seek(msgId);
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void seek(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) {
- seek(topicPartition, offsetAndMetadata.offset());
- }
-
- @Override
- public void seekToBeginning(Collection<TopicPartition> partitions) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- if (partitions.isEmpty()) {
- partitions = consumers.keySet();
- }
- lastCommittedOffset.clear();
- lastReceivedOffset.clear();
-
- for (TopicPartition tp : partitions) {
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(tp);
- if (c == null) {
- futures.add(FutureUtil.failedFuture(
- new IllegalArgumentException("Cannot seek on a partition where we are not subscribed")));
- } else {
- futures.add(c.seekAsync(MessageId.earliest));
- }
- }
-
- FutureUtil.waitForAll(futures).join();
- }
-
- @Override
- public void seekToEnd(Collection<TopicPartition> partitions) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- if (partitions.isEmpty()) {
- partitions = consumers.keySet();
- }
- lastCommittedOffset.clear();
- lastReceivedOffset.clear();
-
- for (TopicPartition tp : partitions) {
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(tp);
- if (c == null) {
- futures.add(FutureUtil.failedFuture(
- new IllegalArgumentException("Cannot seek on a partition where we are not subscribed")));
- } else {
- futures.add(c.seekAsync(MessageId.latest));
- }
- }
-
- FutureUtil.waitForAll(futures).join();
- }
-
- @Override
- public long position(TopicPartition partition) {
- Long offset = lastReceivedOffset.get(partition);
- if (offset == null && !unpolledPartitions.contains(partition)) {
- return resetOffsets(partition).getValue();
- }
- return unpolledPartitions.contains(partition) ? 0 : offset;
- }
-
- @Override
- public long position(TopicPartition topicPartition, Duration duration) {
- return position(topicPartition);
- }
-
- private SubscriptionInitialPosition resetOffsets(final TopicPartition partition) {
- log.info("Resetting partition {} and seeking to {} position", partition, strategy);
- if (strategy == SubscriptionInitialPosition.Earliest) {
- seekToBeginning(Collections.singleton(partition));
- } else {
- seekToEnd(Collections.singleton(partition));
- }
- return strategy;
- }
-
- @Override
- public OffsetAndMetadata committed(TopicPartition partition) {
- return lastCommittedOffset.get(partition);
- }
-
- @Override
- public OffsetAndMetadata committed(TopicPartition topicPartition, Duration duration) {
- return committed(topicPartition);
- }
-
- @Override
- public Map<MetricName, ? extends Metric> metrics() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<PartitionInfo> partitionsFor(String topic) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<PartitionInfo> partitionsFor(String s, Duration duration) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<String, List<PartitionInfo>> listTopics() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<String, List<PartitionInfo>> listTopics(Duration duration) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Set<TopicPartition> paused() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void pause(Collection<TopicPartition> partitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void resume(Collection<TopicPartition> partitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(Map<TopicPartition, Long> timestampsToSearch) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(Map<TopicPartition, Long> map, Duration duration) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition> partitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition> collection, Duration duration) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, Long> endOffsets(Collection<TopicPartition> partitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<TopicPartition, Long> endOffsets(Collection<TopicPartition> collection, Duration duration) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- close(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
- }
-
- @Override
- public void close(long timeout, TimeUnit unit) {
- try {
- closed = true;
-
- if (isAutoCommit) {
- commitAsync();
- }
-
- client.closeAsync().get(timeout, unit);
- } catch (InterruptedException | ExecutionException | TimeoutException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close(Duration duration) {
- close(duration.toMillis(), TimeUnit.MILLISECONDS);
- }
-
- @Override
- public void wakeup() {
- throw new UnsupportedOperationException();
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
deleted file mode 100644
index 13f97b5..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import java.nio.charset.StandardCharsets;
-import java.time.Duration;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.ProducerFencedException;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.CompressionType;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarKafkaSchema;
-import org.apache.pulsar.client.kafka.compat.PulsarProducerKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.KafkaMessageRouter;
-import org.apache.pulsar.client.kafka.compat.KafkaProducerInterceptorWrapper;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PulsarKafkaProducer<K, V> implements Producer<K, V> {
-
- private final PulsarClient client;
- private final ProducerBuilder<byte[]> pulsarProducerBuilder;
-
- private final ConcurrentMap<String, org.apache.pulsar.client.api.Producer<byte[]>> producers = new ConcurrentHashMap<>();
-
- private final Schema<K> keySchema;
- private final Schema<V> valueSchema;
-
- private final Partitioner partitioner;
- private volatile Cluster cluster = Cluster.empty();
-
- private List<ProducerInterceptor<K, V>> interceptors;
-
- private final Properties properties;
-
- public PulsarKafkaProducer(Map<String, Object> configs) {
- this(new ProducerConfig(configs), null, null);
- }
-
- public PulsarKafkaProducer(Map<String, Object> configs, Serializer<K> keySerializer,
- Serializer<V> valueSerializer) {
- this(new ProducerConfig(configs), new PulsarKafkaSchema<>(keySerializer), new PulsarKafkaSchema<>(valueSerializer));
- }
-
- public PulsarKafkaProducer(Map<String, Object> configs, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ProducerConfig(configs), keySchema, valueSchema);
- }
-
- public PulsarKafkaProducer(Properties properties) {
- this(new ProducerConfig(properties), null, null);
- }
-
- public PulsarKafkaProducer(Properties properties, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
- this(new ProducerConfig(properties), new PulsarKafkaSchema<>(keySerializer), new PulsarKafkaSchema<>(valueSerializer));
- }
-
- public PulsarKafkaProducer(Properties properties, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ProducerConfig(properties), keySchema, valueSchema);
- }
-
- @SuppressWarnings({ "unchecked", "deprecation" })
- private PulsarKafkaProducer(ProducerConfig producerConfig, Schema<K> keySchema, Schema<V> valueSchema) {
-
- if (keySchema == null) {
- Serializer<K> kafkaKeySerializer = producerConfig.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class);
- kafkaKeySerializer.configure(producerConfig.originals(), true);
- this.keySchema = new PulsarKafkaSchema<>(kafkaKeySerializer);
- } else {
- this.keySchema = keySchema;
- producerConfig.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
- }
-
- if (valueSchema == null) {
- Serializer<V> kafkaValueSerializer = producerConfig.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class);
- kafkaValueSerializer.configure(producerConfig.originals(), false);
- this.valueSchema = new PulsarKafkaSchema<>(kafkaValueSerializer);
- } else {
- this.valueSchema = valueSchema;
- producerConfig.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
- }
-
- partitioner = producerConfig.getConfiguredInstance(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioner.class);
- partitioner.configure(producerConfig.originals());
-
- this.properties = new Properties();
- producerConfig.originals().forEach((k, v) -> properties.put(k, v));
-
- long keepAliveIntervalMs = Long.parseLong(properties.getProperty(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "30000"));
-
- String serviceUrl = producerConfig.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG).get(0);
- try {
- // Support Kafka's ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG in ms.
- // If passed in value is greater than Integer.MAX_VALUE in second will throw IllegalArgumentException.
- int keepAliveInterval = Math.toIntExact(keepAliveIntervalMs / 1000);
- client = PulsarClientKafkaConfig.getClientBuilder(properties).serviceUrl(serviceUrl).keepAliveInterval(keepAliveInterval, TimeUnit.SECONDS).build();
- } catch (ArithmeticException e) {
- String errorMessage = String.format("Invalid value %d for 'connections.max.idle.ms'. Please use a value smaller than %d000 milliseconds.", keepAliveIntervalMs, Integer.MAX_VALUE);
- logger.error(errorMessage);
- throw new IllegalArgumentException(errorMessage);
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
-
- pulsarProducerBuilder = PulsarProducerKafkaConfig.getProducerBuilder(client, properties);
-
- // To mimic the same batching mode as Kafka, we need to wait a very little amount of
- // time to batch if the client is trying to send messages fast enough
- long lingerMs = Long.parseLong(properties.getProperty(ProducerConfig.LINGER_MS_CONFIG, "1"));
- pulsarProducerBuilder.batchingMaxPublishDelay(lingerMs, TimeUnit.MILLISECONDS);
-
- String compressionType = properties.getProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG);
- if ("gzip".equals(compressionType)) {
- pulsarProducerBuilder.compressionType(CompressionType.ZLIB);
- } else if ("lz4".equals(compressionType)) {
- pulsarProducerBuilder.compressionType(CompressionType.LZ4);
- }
-
- pulsarProducerBuilder.messageRouter(new KafkaMessageRouter(lingerMs));
-
- int sendTimeoutMillis = Integer.parseInt(properties.getProperty(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "30000"));
- pulsarProducerBuilder.sendTimeout(sendTimeoutMillis, TimeUnit.MILLISECONDS);
-
- // Kafka blocking semantic when blockOnBufferFull=false is different from Pulsar client
- // Pulsar throws error immediately when the queue is full and blockIfQueueFull=false
- // Kafka, on the other hand, still blocks for "max.block.ms" time and then gives error.
- Boolean sendTimeOutConfigured = sendTimeoutMillis > 0;
- boolean shouldBlockPulsarProducer = Boolean.getBoolean(properties
- .getProperty(PulsarProducerKafkaConfig.BLOCK_IF_PRODUCER_QUEUE_FULL, sendTimeOutConfigured.toString()));
- pulsarProducerBuilder.blockIfQueueFull(shouldBlockPulsarProducer);
-
- interceptors = (List) producerConfig.getConfiguredInstances(
- ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, ProducerInterceptor.class);
- }
-
- @Override
- public void initTransactions() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void beginTransaction() throws ProducerFencedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> map, String s) throws ProducerFencedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void commitTransaction() throws ProducerFencedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void abortTransaction() throws ProducerFencedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Future<RecordMetadata> send(ProducerRecord<K, V> record) {
- return send(record, null);
- }
-
- @Override
- public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
- org.apache.pulsar.client.api.Producer<byte[]> producer;
-
- try {
- producer = producers.computeIfAbsent(record.topic(), topic -> createNewProducer(topic));
- } catch (Exception e) {
- if (callback != null) {
- callback.onCompletion(null, e);
- }
- CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
- }
-
- TypedMessageBuilder<byte[]> messageBuilder = producer.newMessage();
- int messageSize = buildMessage(messageBuilder, record);
-
- CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
- messageBuilder.sendAsync().thenAccept((messageId) -> {
- future.complete(getRecordMetadata(record.topic(), messageBuilder, messageId, messageSize));
- }).exceptionally(ex -> {
- future.completeExceptionally(ex);
- return null;
- });
-
- future.handle((recordMetadata, throwable) -> {
- if (callback != null) {
- Exception exception = throwable != null ? new Exception(throwable) : null;
- callback.onCompletion(recordMetadata, exception);
- }
- return null;
- });
-
- return future;
- }
-
- @Override
- public void flush() {
- producers.values().stream()
- .map(p -> p.flushAsync())
- .collect(Collectors.toList())
- .forEach(CompletableFuture::join);
- }
-
- @Override
- public List<PartitionInfo> partitionsFor(String topic) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<MetricName, ? extends Metric> metrics() {
- return Collections.emptyMap();
- }
-
- @Override
- public void close() {
- close(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
- partitioner.close();
- }
-
- @Override
- public void close(long timeout, TimeUnit unit) {
- try {
- client.closeAsync().get(timeout, unit);
- } catch (InterruptedException | ExecutionException | TimeoutException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close(Duration duration) {
- close(duration.toMillis(), TimeUnit.MILLISECONDS);
- }
-
- private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
- try {
- // Add the partitions info for the new topic
- synchronized (this){
- cluster = cluster.withPartitions(readPartitionsInfo(topic));
- }
- List<org.apache.pulsar.client.api.ProducerInterceptor> wrappedInterceptors = interceptors.stream()
- .map(interceptor -> new KafkaProducerInterceptorWrapper(interceptor, keySchema, valueSchema, topic))
- .collect(Collectors.toList());
- return pulsarProducerBuilder.clone()
- .topic(topic)
- .intercept(wrappedInterceptors.toArray(new org.apache.pulsar.client.api.ProducerInterceptor[wrappedInterceptors.size()]))
- .create();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- private Map<TopicPartition, PartitionInfo> readPartitionsInfo(String topic) {
- List<String> partitions = client.getPartitionsForTopic(topic).join();
-
- Map<TopicPartition, PartitionInfo> partitionsInfo = new HashMap<>();
-
- for (int i = 0; i < partitions.size(); i++) {
- TopicPartition tp = new TopicPartition(topic, i);
- PartitionInfo pi = new PartitionInfo(topic, i, null, null, null);
- partitionsInfo.put(tp, pi);
- }
-
- return partitionsInfo;
- }
-
- private int buildMessage(TypedMessageBuilder<byte[]> builder, ProducerRecord<K, V> record) {
- byte[] keyBytes = null;
- if (record.key() != null) {
- String key = getKey(record.topic(), record.key());
- keyBytes = key.getBytes(StandardCharsets.UTF_8);
- builder.key(key);
- }
-
- if (record.timestamp() != null) {
- builder.eventTime(record.timestamp());
- }
-
- if (valueSchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<V>) valueSchema).setTopic(record.topic());
- }
- byte[] value = valueSchema.encode(record.value());
- builder.value(value);
-
- if (record.partition() != null) {
- // Partition was explicitly set on the record
- builder.property(KafkaMessageRouter.PARTITION_ID, record.partition().toString());
- } else {
- // Get the partition id from the partitioner
- int partition = partitioner.partition(record.topic(), record.key(), keyBytes, record.value(), value, cluster);
- builder.property(KafkaMessageRouter.PARTITION_ID, Integer.toString(partition));
- }
-
- return value.length;
- }
-
- private String getKey(String topic, K key) {
- // If key is a String, we can use it as it is, otherwise, serialize to byte[] and encode in base64
- if (key instanceof String) {
- return (String) key;
- }
- if (keySchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema) keySchema).setTopic(topic);
- }
- byte[] keyBytes = keySchema.encode(key);
- return Base64.getEncoder().encodeToString(keyBytes);
- }
-
- private RecordMetadata getRecordMetadata(String topic, TypedMessageBuilder<byte[]> msgBuilder, MessageId messageId,
- int size) {
- MessageIdImpl msgId = (MessageIdImpl) messageId;
-
- // Combine ledger id and entry id to form offset
- long offset = MessageIdUtils.getOffset(msgId);
- int partition = msgId.getPartitionIndex();
-
- TopicPartition tp = new TopicPartition(topic, partition);
- TypedMessageBuilderImpl<byte[]> mb = (TypedMessageBuilderImpl<byte[]>) msgBuilder;
- return new RecordMetadata(tp, offset, 0L, mb.getPublishTime(), 0L, mb.hasKey() ? mb.getKey().length() : 0, size);
- }
-
- private static final Logger logger = LoggerFactory.getLogger(PulsarKafkaProducer.class);
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java
deleted file mode 100644
index d3fb915..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.concurrent.ThreadLocalRandom;
-
-import org.apache.pulsar.client.api.HashingScheme;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.TopicMetadata;
-import org.apache.pulsar.client.impl.RoundRobinPartitionMessageRouterImpl;
-
-public class KafkaMessageRouter extends RoundRobinPartitionMessageRouterImpl {
-
- public static final String PARTITION_ID = "pulsar.partition.id";
-
- public KafkaMessageRouter(long maxBatchingDelayMs) {
- super(HashingScheme.JavaStringHash, ThreadLocalRandom.current().nextInt(), true, maxBatchingDelayMs);
- }
-
- @Override
- public int choosePartition(Message<?> msg, TopicMetadata metadata) {
- if (msg.hasProperty(PARTITION_ID)) {
- return Integer.parseInt(msg.getProperty(PARTITION_ID));
- } else {
- return super.choosePartition(msg, metadata);
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapper.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapper.java
deleted file mode 100644
index dbe827f..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapper.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.apache.kafka.common.serialization.ByteBufferDeserializer;
-import org.apache.kafka.common.serialization.ByteBufferSerializer;
-import org.apache.kafka.common.serialization.BytesDeserializer;
-import org.apache.kafka.common.serialization.BytesSerializer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.DoubleDeserializer;
-import org.apache.kafka.common.serialization.DoubleSerializer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.LongDeserializer;
-import org.apache.kafka.common.serialization.LongSerializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerInterceptor;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.MessageImpl;
-import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
-import org.apache.pulsar.common.api.proto.PulsarApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Base64;
-
-/**
- * A wrapper for Kafka's {@link org.apache.kafka.clients.producer.ProducerInterceptor} to make pulsar support
- * Kafka ProducerInterceptor. It holds an instance of {@link org.apache.kafka.clients.producer.ProducerInterceptor}
- * and it'll delegate all invocation to that instance.
- * <p>
- * Extend {@link ProducerInterceptor<byte[]>} as all Pulsar {@link Message} created by
- * {@link org.apache.kafka.clients.producer.PulsarKafkaProducer} is of type byte[].
- *
- */
-public class KafkaProducerInterceptorWrapper<K, V> implements ProducerInterceptor<byte[]> {
-
- private static final Logger log = LoggerFactory.getLogger(KafkaProducerInterceptorWrapper.class);
-
- final private org.apache.kafka.clients.producer.ProducerInterceptor<K, V> kafkaProducerInterceptor;
-
- // For serializer key/value, and to determine the deserializer for key/value.
- private final Schema<K> keySchema;
-
- private final Schema<V> valueSchema;
-
- // Keep the topic, as each Pulsar producer will tie to a Kafka topic, and ProducerInterceptor will tie to a Pulsar
- // producer, it's safe to set it as final.
- private final String topic;
-
- private Schema<byte[]> scheme;
-
- private long eventTime;
-
- private String partitionID;
-
- /**
- * Create a wrapper of type {@link ProducerInterceptor} that will delegate all work to underlying Kafka's interceptor.
- *
- * @param kafkaProducerInterceptor Underlying instance of {@link org.apache.kafka.clients.producer.ProducerInterceptor<K, V>}
- * that this wrapper will delegate work to.
- * @param keySerializer {@link Serializer} used to serialize Kafka {@link ProducerRecord#key}.
- * @param valueSerializer {@link Serializer} used to serialize Kafka {@link ProducerRecord#value}.
- * @param topic Topic this {@link ProducerInterceptor} will be associated to.
- */
- public KafkaProducerInterceptorWrapper(org.apache.kafka.clients.producer.ProducerInterceptor<K, V> kafkaProducerInterceptor,
- Schema<K> keySchema,
- Schema<V> valueSchema,
- String topic) {
- this.kafkaProducerInterceptor = kafkaProducerInterceptor;
- this.keySchema = keySchema;
- this.valueSchema = valueSchema;
- this.topic = topic;
- }
-
- /**
- * Called when interceptor is closed.
- * The wrapper itself doesn't own any resource, just call underlying {@link org.apache.kafka.clients.producer.ProducerInterceptor#close()}
- */
- @Override
- public void close() {
- kafkaProducerInterceptor.close();
- }
-
- /**
- * It tries to convert a Pulsar {@link Message} to a Kafka{@link ProducerRecord}, pass it to underlying
- * {@link org.apache.kafka.clients.producer.ProducerInterceptor#onSend(ProducerRecord)} then convert the output
- * back to Pulsar {@link Message}.
- * <p>
- * When build a Pulsar {@link Message} at {@link org.apache.kafka.clients.producer.PulsarKafkaProducer#buildMessage}
- * schema, eventtime, partitionID, key and value are set. All this information will be preserved during the conversion.
- *
- * @param producer the producer which contains the interceptor, will be ignored as Kafka
- * {@link org.apache.kafka.clients.producer.ProducerInterceptor} doesn't use it.
- * @param message message to send
- * @return Processed message.
- */
- @Override
- public Message<byte[]> beforeSend(Producer<byte[]> producer, Message<byte[]> message) {
- return toPulsarMessage(kafkaProducerInterceptor.onSend(toKafkaRecord(message)));
- }
-
- /**
- * Delegate work to {@link org.apache.kafka.clients.producer.ProducerInterceptor#onAcknowledgement}
- * @param producer the producer which contains the interceptor.
- * @param message the message that application sends
- * @param msgId the message id that assigned by the broker; null if send failed.
- * @param exception the exception on sending messages, null indicates send has succeed.
- */
- @Override
- public void onSendAcknowledgement(Producer<byte[]> producer, Message<byte[]> message, MessageId msgId, Throwable exception) {
- try {
- PulsarApi.MessageMetadata.Builder messageMetadataBuilder = ((MessageImpl<byte[]>)message).getMessageBuilder();
- partitionID = getPartitionID(messageMetadataBuilder);
- TopicPartition topicPartition = new TopicPartition(topic, Integer.parseInt(partitionID));
- kafkaProducerInterceptor.onAcknowledgement(new RecordMetadata(topicPartition,
- -1L,
- -1L,
- messageMetadataBuilder.getEventTime(),
- -1L,
- message.getKeyBytes().length,
- message.getValue().length), new Exception(exception));
- } catch (NumberFormatException e) {
- String errorMessage = "Unable to convert partitionID to integer: " + e.getMessage();
- log.error(errorMessage);
- throw new RuntimeException(errorMessage);
- }
- }
-
- /**
- * Convert a Kafka {@link ProducerRecord} to a Pulsar {@link Message}.
- *
- * @param producerRecord Kafka record to be convert.
- * @return Pulsar message.
- */
- private Message<byte[]> toPulsarMessage(ProducerRecord<K, V> producerRecord) {
- TypedMessageBuilderImpl typedMessageBuilder = new TypedMessageBuilderImpl(null, scheme);
- typedMessageBuilder.key(serializeKey(topic, producerRecord.key()));
- if (valueSchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<V>) valueSchema).setTopic(topic);
- }
- typedMessageBuilder.value(valueSchema.encode(producerRecord.value()));
- typedMessageBuilder.eventTime(eventTime);
- typedMessageBuilder.property(KafkaMessageRouter.PARTITION_ID, partitionID);
- return typedMessageBuilder.getMessage();
- }
-
- /**
- * Convert a Pulsar {@link Message} to a Kafka {@link ProducerRecord}.
- * First it'll store those field that Kafka record doesn't need such as schema.
- * Then it try to deserialize the value as it's been serialized to byte[] when creating the message.
- *
- * @param message Pulsar message to be convert.
- * @return Kafka record.
- */
- private ProducerRecord<K, V> toKafkaRecord(Message<byte[]> message) {
- V value;
- if (valueSchema instanceof PulsarKafkaSchema) {
- PulsarKafkaSchema<V> pulsarKeyKafkaSchema = (PulsarKafkaSchema<V>) valueSchema;
- Deserializer valueDeserializer = getDeserializer((pulsarKeyKafkaSchema.getKafkaSerializer()));
- value = (V) valueDeserializer.deserialize(topic, message.getValue());
- } else {
- value = valueSchema.decode(message.getValue());
- }
- try {
- scheme = (Schema<byte[]>) FieldUtils.readField(message, "schema", true);
- PulsarApi.MessageMetadata.Builder messageMetadataBuilder = ((MessageImpl<byte[]>)message).getMessageBuilder();
- partitionID = getPartitionID(messageMetadataBuilder);
- eventTime = message.getEventTime();
- return new ProducerRecord<>(topic, Integer.parseInt(partitionID), eventTime, deserializeKey(topic, message.getKey()), value);
- } catch (NumberFormatException e) {
- // If not able to parse partitionID, ignore it.
- return new ProducerRecord<>(topic, deserializeKey(topic, message.getKey()), value);
- } catch (IllegalAccessException e) {
- String errorMessage = "Unable to get the schema of message due to " + e.getMessage();
- log.error(errorMessage);
- throw new RuntimeException(errorMessage);
- }
- }
-
- private String serializeKey(String topic, K key) {
- // If key is a String, we can use it as it is, otherwise, serialize to byte[] and encode in base64
- if (key instanceof String) {
- return (String) key;
- }
- if (keySchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<K>) keySchema).setTopic(topic);
- }
- byte[] keyBytes = keySchema.encode(key);
- return Base64.getEncoder().encodeToString(keyBytes);
- }
-
- private K deserializeKey(String topic, String key) {
- if (keySchema instanceof PulsarKafkaSchema) {
- PulsarKafkaSchema<K> pulsarKeyKafkaSchema = (PulsarKafkaSchema<K>) keySchema;
- // If key is a String, we can use it as it is, otherwise, serialize to byte[] and encode in base64
- if (pulsarKeyKafkaSchema.getKafkaSerializer() instanceof StringSerializer) {
- return (K) key;
- }
-
- Deserializer keyDeserializer = getDeserializer(pulsarKeyKafkaSchema.getKafkaSerializer());
- return (K) keyDeserializer.deserialize(topic, Base64.getDecoder().decode(key));
- }
- return keySchema.decode(Base64.getDecoder().decode(key));
- }
-
- /**
- * Try to get the partitionID from messageMetadataBuilder.
- * As it is set in {@link org.apache.kafka.clients.producer.PulsarKafkaProducer#buildMessage}, can guarantee
- * a partitionID will be return.
- *
- * @param messageMetadataBuilder
- * @return PartitionID
- */
- private String getPartitionID(PulsarApi.MessageMetadata.Builder messageMetadataBuilder) {
- return messageMetadataBuilder.getPropertiesList()
- .stream()
- .filter(keyValue -> keyValue.getKey().equals(KafkaMessageRouter.PARTITION_ID))
- .findFirst()
- .get()
- .getValue();
- }
-
- static Deserializer getDeserializer(Serializer serializer) {
- if (serializer instanceof StringSerializer) {
- return new StringDeserializer();
- } else if (serializer instanceof LongSerializer) {
- return new LongDeserializer();
- } else if (serializer instanceof IntegerSerializer) {
- return new IntegerDeserializer();
- } else if (serializer instanceof DoubleSerializer) {
- return new DoubleDeserializer();
- } else if (serializer instanceof BytesSerializer) {
- return new BytesDeserializer();
- } else if (serializer instanceof ByteBufferSerializer) {
- return new ByteBufferDeserializer();
- } else if (serializer instanceof ByteArraySerializer) {
- return new ByteArrayDeserializer();
- } else {
- throw new IllegalArgumentException(serializer.getClass().getName() + " is not a valid or supported subclass of org.apache.kafka.common.serialization.Serializer.");
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java
deleted file mode 100644
index 8ba5ede..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-
-public class PulsarClientKafkaConfig {
-
- /// Config variables
- public static final String AUTHENTICATION_CLASS = "pulsar.authentication.class";
- public static final String AUTHENTICATION_PARAMS_MAP = "pulsar.authentication.params.map";
- public static final String AUTHENTICATION_PARAMS_STRING = "pulsar.authentication.params.string";
- public static final String USE_TLS = "pulsar.use.tls";
- public static final String TLS_TRUST_CERTS_FILE_PATH = "pulsar.tls.trust.certs.file.path";
- public static final String TLS_ALLOW_INSECURE_CONNECTION = "pulsar.tls.allow.insecure.connection";
- public static final String TLS_HOSTNAME_VERIFICATION = "pulsar.tls.hostname.verification";
-
- public static final String OPERATION_TIMEOUT_MS = "pulsar.operation.timeout.ms";
- public static final String STATS_INTERVAL_SECONDS = "pulsar.stats.interval.seconds";
- public static final String NUM_IO_THREADS = "pulsar.num.io.threads";
-
- public static final String CONNECTIONS_PER_BROKER = "pulsar.connections.per.broker";
-
- public static final String USE_TCP_NODELAY = "pulsar.use.tcp.nodelay";
-
- public static final String CONCURRENT_LOOKUP_REQUESTS = "pulsar.concurrent.lookup.requests";
- public static final String MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION = "pulsar.max.number.rejected.request.per.connection";
-
- public static final String KEEPALIVE_INTERVAL_MS = "pulsar.keepalive.interval.ms";
-
- public static ClientBuilder getClientBuilder(Properties properties) {
- ClientBuilder clientBuilder = PulsarClient.builder();
-
- if (properties.containsKey(AUTHENTICATION_CLASS)) {
- String className = properties.getProperty(AUTHENTICATION_CLASS);
- try {
- if (properties.containsKey(AUTHENTICATION_PARAMS_STRING)) {
- String authParamsString = (String) properties.get(AUTHENTICATION_PARAMS_STRING);
- clientBuilder.authentication(className, authParamsString);
- } else if (properties.containsKey(AUTHENTICATION_PARAMS_MAP)) {
- Map<String, String> authParams = (Map<String, String>) properties.get(AUTHENTICATION_PARAMS_MAP);
- clientBuilder.authentication(className, authParams);
- } else {
- @SuppressWarnings("unchecked")
- Class<Authentication> clazz = (Class<Authentication>) Class.forName(className);
- Authentication auth = clazz.newInstance();
- clientBuilder.authentication(auth);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- clientBuilder.enableTls(Boolean.parseBoolean(properties.getProperty(USE_TLS, "false")));
- clientBuilder.allowTlsInsecureConnection(
- Boolean.parseBoolean(properties.getProperty(TLS_ALLOW_INSECURE_CONNECTION, "false")));
- clientBuilder.enableTlsHostnameVerification(
- Boolean.parseBoolean(properties.getProperty(TLS_HOSTNAME_VERIFICATION, "false")));
-
- if (properties.containsKey(TLS_TRUST_CERTS_FILE_PATH)) {
- clientBuilder.tlsTrustCertsFilePath(properties.getProperty(TLS_TRUST_CERTS_FILE_PATH));
- }
-
- if (properties.containsKey(OPERATION_TIMEOUT_MS)) {
- clientBuilder.operationTimeout(Integer.parseInt(properties.getProperty(OPERATION_TIMEOUT_MS)),
- TimeUnit.MILLISECONDS);
- }
-
- if (properties.containsKey(STATS_INTERVAL_SECONDS)) {
- clientBuilder.statsInterval(Integer.parseInt(properties.getProperty(STATS_INTERVAL_SECONDS)),
- TimeUnit.SECONDS);
- }
-
- if (properties.containsKey(NUM_IO_THREADS)) {
- clientBuilder.ioThreads(Integer.parseInt(properties.getProperty(NUM_IO_THREADS)));
- }
-
- if (properties.containsKey(CONNECTIONS_PER_BROKER)) {
- clientBuilder.connectionsPerBroker(Integer.parseInt(properties.getProperty(CONNECTIONS_PER_BROKER)));
- }
-
- if (properties.containsKey(USE_TCP_NODELAY)) {
- clientBuilder.enableTcpNoDelay(Boolean.parseBoolean(properties.getProperty(USE_TCP_NODELAY)));
- }
-
- if (properties.containsKey(CONCURRENT_LOOKUP_REQUESTS)) {
- clientBuilder.maxConcurrentLookupRequests(Integer.parseInt(properties.getProperty(CONCURRENT_LOOKUP_REQUESTS)));
- }
-
- if (properties.containsKey(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)) {
- clientBuilder.maxNumberOfRejectedRequestPerConnection(
- Integer.parseInt(properties.getProperty(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)));
- }
-
- if (properties.containsKey(KEEPALIVE_INTERVAL_MS)) {
- clientBuilder.keepAliveInterval(Integer.parseInt(properties.getProperty(KEEPALIVE_INTERVAL_MS)),
- TimeUnit.MILLISECONDS);
- }
-
- return clientBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java
deleted file mode 100644
index a527827..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Arrays;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.RegexSubscriptionMode;
-
-public class PulsarConsumerKafkaConfig {
-
- /// Config variables
- public static final String CONSUMER_NAME = "pulsar.consumer.name";
- public static final String RECEIVER_QUEUE_SIZE = "pulsar.consumer.receiver.queue.size";
- public static final String ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS = "pulsar.consumer.acknowledgments.group.time.millis";
- public static final String TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS = "pulsar.consumer.total.receiver.queue.size.across.partitions";
- public static final String SUBSCRIPTION_TOPICS_MODE = "pulsar.consumer.subscription.topics.mode";
-
- public static ConsumerBuilder<byte[]> getConsumerBuilder(PulsarClient client, Properties properties) {
- ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer();
-
- if (properties.containsKey(CONSUMER_NAME)) {
- consumerBuilder.consumerName(properties.getProperty(CONSUMER_NAME));
- }
-
- if (properties.containsKey(RECEIVER_QUEUE_SIZE)) {
- consumerBuilder.receiverQueueSize(Integer.parseInt(properties.getProperty(RECEIVER_QUEUE_SIZE)));
- }
-
- if (properties.containsKey(TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS)) {
- consumerBuilder.maxTotalReceiverQueueSizeAcrossPartitions(
- Integer.parseInt(properties.getProperty(TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS)));
- }
-
- if (properties.containsKey(ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS)) {
- consumerBuilder.acknowledgmentGroupTime(
- Long.parseLong(properties.getProperty(ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS)), TimeUnit.MILLISECONDS);
- }
-
- if (properties.containsKey(SUBSCRIPTION_TOPICS_MODE)) {
- RegexSubscriptionMode mode;
- try {
- mode = RegexSubscriptionMode.valueOf(properties.getProperty(SUBSCRIPTION_TOPICS_MODE));
- } catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("Illegal subscription mode, valid values are: "
- + Arrays.asList(RegexSubscriptionMode.values()));
- }
- consumerBuilder.subscriptionTopicsMode(mode);
- }
-
- return consumerBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java
deleted file mode 100644
index aef6dd1..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-public class PulsarKafkaSchema<T> implements Schema<T> {
-
- private final Serializer<T> kafkaSerializer;
-
- private final Deserializer<T> kafkaDeserializer;
-
- private String topic;
-
- public PulsarKafkaSchema(Serializer<T> serializer) {
- this(serializer, null);
- }
-
- public PulsarKafkaSchema(Deserializer<T> deserializer) {
- this(null, deserializer);
- }
-
- public PulsarKafkaSchema(Serializer<T> serializer, Deserializer<T> deserializer) {
- this.kafkaSerializer = serializer;
- this.kafkaDeserializer = deserializer;
- }
-
- public Serializer<T> getKafkaSerializer() {
- return kafkaSerializer;
- }
-
- public Deserializer<T> getKafkaDeserializer() {
- return kafkaDeserializer;
- }
-
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- @Override
- public byte[] encode(T message) {
- checkArgument(kafkaSerializer != null, "Kafka serializer is not initialized yet");
- return kafkaSerializer.serialize(this.topic, message);
- }
-
- @Override
- public T decode(byte[] message) {
- checkArgument(kafkaDeserializer != null, "Kafka deserializer is not initialized yet");
- return kafkaDeserializer.deserialize(this.topic, message);
- }
-
- @Override
- public SchemaInfo getSchemaInfo() {
- return Schema.BYTES.getSchemaInfo();
- }
-
- @Override
- public Schema<T> clone() {
- return this;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java
deleted file mode 100644
index 7554faf..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Properties;
-
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException.ProducerQueueIsFullError;
-
-public class PulsarProducerKafkaConfig {
-
- /// Config variables
- public static final String PRODUCER_NAME = "pulsar.producer.name";
- public static final String INITIAL_SEQUENCE_ID = "pulsar.producer.initial.sequence.id";
-
- public static final String MAX_PENDING_MESSAGES = "pulsar.producer.max.pending.messages";
- public static final String MAX_PENDING_MESSAGES_ACROSS_PARTITIONS = "pulsar.producer.max.pending.messages.across.partitions";
- public static final String BATCHING_ENABLED = "pulsar.producer.batching.enabled";
- public static final String BATCHING_MAX_MESSAGES = "pulsar.producer.batching.max.messages";
- /**
- * send operations will immediately fail with {@link ProducerQueueIsFullError} when there is no space left in
- * pending queue.
- **/
- public static final String BLOCK_IF_PRODUCER_QUEUE_FULL = "pulsar.block.if.producer.queue.full";
-
- public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
- ProducerBuilder<byte[]> producerBuilder = client.newProducer();
-
- if (properties.containsKey(PRODUCER_NAME)) {
- producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
- }
-
- if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
- producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
- }
-
- if (properties.containsKey(MAX_PENDING_MESSAGES)) {
- producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
- }
-
- if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
- producerBuilder.maxPendingMessagesAcrossPartitions(
- Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
- }
-
- producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));
-
- if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
- producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
- }
-
- return producerBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
deleted file mode 100644
index b1069ad6..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-import org.apache.avro.reflect.Nullable;
-import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarProducerKafkaConfig;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.testng.Assert;
-import org.testng.IObjectFactory;
-import org.testng.annotations.ObjectFactory;
-import org.testng.annotations.Test;
-
-@PrepareForTest({PulsarClientKafkaConfig.class, PulsarProducerKafkaConfig.class})
-@PowerMockIgnore({"org.apache.logging.log4j.*", "org.apache.kafka.clients.producer.ProducerInterceptor"})
-public class PulsarKafkaProducerTest {
-
- @Data
- @ToString
- @EqualsAndHashCode
- public static class Foo {
- @Nullable
- private String field1;
- @Nullable
- private String field2;
- private int field3;
- }
-
- @Data
- @ToString
- @EqualsAndHashCode
- public static class Bar {
- private boolean field1;
- }
-
- @ObjectFactory
- // Necessary to make PowerMockito.mockStatic work with TestNG.
- public IObjectFactory getObjectFactory() {
- return new org.powermock.modules.testng.PowerMockObjectFactory();
- }
-
- @Test
- public void testPulsarKafkaProducer() {
- ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
- ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
- doAnswer(invocation -> {
- Assert.assertEquals((int)invocation.getArguments()[0], 1000000, "Send time out is suppose to be 1000.");
- return mockProducerBuilder;
- }).when(mockProducerBuilder).sendTimeout(anyInt(), any(TimeUnit.class));
- doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
- doAnswer(invocation -> {
- Assert.assertEquals((int)invocation.getArguments()[0], 1000, "Keep alive interval is suppose to be 1000.");
- return mockClientBuilder;
- }).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));
-
- PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
- PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
- when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
- when(PulsarProducerKafkaConfig.getProducerBuilder(any(), any())).thenReturn(mockProducerBuilder);
-
- Properties properties = new Properties();
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
- properties.put(PulsarProducerKafkaConfig.BLOCK_IF_PRODUCER_QUEUE_FULL, Boolean.FALSE.toString());
-
- new PulsarKafkaProducer<>(properties);
-
- verify(mockClientBuilder, times(1)).keepAliveInterval(1000, TimeUnit.SECONDS);
- verify(mockProducerBuilder, times(1)).sendTimeout(1000000, TimeUnit.MILLISECONDS);
- verify(mockProducerBuilder, times(1)).blockIfQueueFull(false);
- }
-
- @Test
- public void testPulsarKafkaInterceptor() throws PulsarClientException {
- // Arrange
- PulsarClient mockClient = mock(PulsarClient.class);
- ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
- org.apache.pulsar.client.api.Producer mockProducer = mock(org.apache.pulsar.client.api.Producer.class);
- ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
- CompletableFuture mockPartitionFuture = new CompletableFuture();
- CompletableFuture mockSendAsyncFuture = new CompletableFuture();
- TypedMessageBuilder mockTypedMessageBuilder = mock(TypedMessageBuilderImpl.class);
-
- mockPartitionFuture.complete(new ArrayList<>());
- mockSendAsyncFuture.complete(new MessageIdImpl(1, 1, 1));
- doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
- doReturn(mockClientBuilder).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));
- doReturn(mockClient).when(mockClientBuilder).build();
- doReturn(mockPartitionFuture).when(mockClient).getPartitionsForTopic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).topic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).clone();
- doReturn(mockProducerBuilder).when(mockProducerBuilder).intercept(
- (org.apache.pulsar.client.api.ProducerInterceptor) any());
- doReturn(mockProducer).when(mockProducerBuilder).create();
- doReturn(mockTypedMessageBuilder).when(mockProducer).newMessage();
- doReturn(mockSendAsyncFuture).when(mockTypedMessageBuilder).sendAsync();
- PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
- PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
- when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
- when(PulsarProducerKafkaConfig.getProducerBuilder(any(PulsarClient.class), any(Properties.class))).thenReturn(mockProducerBuilder);
-
- Properties properties = new Properties();
- List interceptors = new ArrayList();
- interceptors.add("org.apache.kafka.clients.producer.PulsarKafkaProducerTest$PulsarKafkaProducerInterceptor");
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, interceptors);
-
- // Act
- PulsarKafkaProducer<String, String> pulsarKafkaProducer = new PulsarKafkaProducer<>(properties);
-
- pulsarKafkaProducer.send(new ProducerRecord<>("topic", 1,"key", "value"));
-
- // Verify
- verify(mockProducerBuilder, times(1)).intercept(
- (org.apache.pulsar.client.api.ProducerInterceptor)any());
- }
-
- @Test
- public void testPulsarKafkaSendAvro() throws PulsarClientException {
- // Arrange
- PulsarClient mockClient = mock(PulsarClient.class);
- ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
- org.apache.pulsar.client.api.Producer mockProducer = mock(org.apache.pulsar.client.api.Producer.class);
- ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
- CompletableFuture mockPartitionFuture = new CompletableFuture();
- CompletableFuture mockSendAsyncFuture = new CompletableFuture();
- TypedMessageBuilder mockTypedMessageBuilder = mock(TypedMessageBuilderImpl.class);
-
- mockPartitionFuture.complete(new ArrayList<>());
- mockSendAsyncFuture.complete(new MessageIdImpl(1, 1, 1));
- doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
- doReturn(mockClientBuilder).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));
- doReturn(mockClient).when(mockClientBuilder).build();
- doReturn(mockPartitionFuture).when(mockClient).getPartitionsForTopic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).topic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).clone();
- doReturn(mockProducerBuilder).when(mockProducerBuilder).intercept(
- (org.apache.pulsar.client.api.ProducerInterceptor) any());
- doReturn(mockProducer).when(mockProducerBuilder).create();
- doReturn(mockTypedMessageBuilder).when(mockProducer).newMessage();
- doReturn(mockSendAsyncFuture).when(mockTypedMessageBuilder).sendAsync();
- PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
- PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
- when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
- when(PulsarProducerKafkaConfig.getProducerBuilder(any(PulsarClient.class), any(Properties.class))).thenReturn(mockProducerBuilder);
-
- Properties properties = new Properties();
- List interceptors = new ArrayList();
- interceptors.add("org.apache.kafka.clients.producer.PulsarKafkaProducerTest$PulsarKafkaProducerInterceptor");
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, interceptors);
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
- // Act
- PulsarKafkaProducer<Foo, Bar> pulsarKafkaProducer = new PulsarKafkaProducer<>(properties, fooSchema, barSchema);
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
- pulsarKafkaProducer.send(new ProducerRecord<>("topic", 1,foo, bar));
-
- // Verify
- verify(mockTypedMessageBuilder, times(1)).sendAsync();
- verify(mockProducerBuilder, times(1)).intercept(
- (org.apache.pulsar.client.api.ProducerInterceptor) any());
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "Invalid value 2147483648000 for 'connections.max.idle.ms'. Please use a value smaller than 2147483647000 milliseconds.")
- public void testPulsarKafkaProducerKeepAliveIntervalIllegalArgumentException() {
- Properties properties = new Properties();
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, Long.toString((Integer.MAX_VALUE + 1L) * 1000));
-
- new PulsarKafkaProducer<>(properties);
- }
-
- public static class PulsarKafkaProducerInterceptor implements org.apache.kafka.clients.producer.ProducerInterceptor<String, String> {
-
- @Override
- public ProducerRecord onSend(ProducerRecord record) {
- return null;
- }
-
- @Override
- public void onAcknowledgement(RecordMetadata metadata, Exception exception) {
-
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public void configure(Map<String, ?> configs) {
-
- }
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapperTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapperTest.java
deleted file mode 100644
index eea92c4..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka/src/test/java/org/apache/pulsar/client/kafka/compat/KafkaProducerInterceptorWrapperTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.stream.Collectors;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.apache.kafka.common.serialization.ByteBufferDeserializer;
-import org.apache.kafka.common.serialization.ByteBufferSerializer;
-import org.apache.kafka.common.serialization.BytesDeserializer;
-import org.apache.kafka.common.serialization.BytesSerializer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.DoubleDeserializer;
-import org.apache.kafka.common.serialization.DoubleSerializer;
-import org.apache.kafka.common.serialization.IntegerDeserializer;
-import org.apache.kafka.common.serialization.IntegerSerializer;
-import org.apache.kafka.common.serialization.LongDeserializer;
-import org.apache.kafka.common.serialization.LongSerializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.ProducerInterceptor;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.interceptor.ProducerInterceptorWrapper;
-import org.apache.pulsar.client.impl.ProducerInterceptors;
-import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
-import org.apache.pulsar.client.impl.schema.BytesSchema;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import java.util.Arrays;
-import java.util.Random;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-
-public class KafkaProducerInterceptorWrapperTest {
-
- /**
- * This test case is to make sure information is not lost during process of convert Pulsar message to Kafka record
- * and back to Pulsar message.
- */
- @Test
- public void testProducerInterceptorConvertRecordCorrectly() {
-
- String topic = "topic name";
- int partitionID = 666;
- long timeStamp = Math.abs(new Random().nextLong());
-
- org.apache.kafka.clients.producer.ProducerInterceptor<String, byte[]> mockInterceptor1 =
- (org.apache.kafka.clients.producer.ProducerInterceptor<String, byte[]>) mock(org.apache.kafka.clients.producer.ProducerInterceptor.class);
-
- doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- ProducerRecord<String, byte[]> record = (ProducerRecord<String, byte[]>) invocation.getArguments()[0];
- assertEquals(record.key(), "original key");
- assertEquals(record.value(), "original value".getBytes());
- assertEquals(record.timestamp().longValue(), timeStamp);
- assertEquals(record.partition().intValue(), partitionID);
- return new ProducerRecord<String, byte[]>(topic, "processed key", "processed value".getBytes());
- }
- }).when(mockInterceptor1).onSend(any(ProducerRecord.class));
-
- org.apache.kafka.clients.producer.ProducerInterceptor<String, String> mockInterceptor2 =
- (org.apache.kafka.clients.producer.ProducerInterceptor<String, String>) mock(org.apache.kafka.clients.producer.ProducerInterceptor.class);
-
- doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- ProducerRecord<String, byte[]> record = (ProducerRecord<String, byte[]>) invocation.getArguments()[0];
- assertEquals(record.key(), "processed key");
- assertEquals(record.value(), "processed value".getBytes());
- assertEquals(record.timestamp().longValue(), timeStamp);
- assertEquals(record.partition().intValue(), partitionID);
- return record;
- }
- }).when(mockInterceptor2).onSend(any(ProducerRecord.class));
-
-
- Schema<String> pulsarKeySerializeSchema = new PulsarKafkaSchema<>(new StringSerializer());
- Schema<byte[]> pulsarValueSerializeSchema = new PulsarKafkaSchema<>(new ByteArraySerializer());
- ProducerInterceptors producerInterceptors = new ProducerInterceptors(
- Arrays.stream(new ProducerInterceptor[]{
- new KafkaProducerInterceptorWrapper(mockInterceptor1, pulsarKeySerializeSchema, pulsarValueSerializeSchema, topic),
- new KafkaProducerInterceptorWrapper(mockInterceptor2, pulsarKeySerializeSchema, pulsarValueSerializeSchema, topic)}).map(
- ProducerInterceptorWrapper::new).collect(Collectors.toList()));
-
- TypedMessageBuilderImpl typedMessageBuilder = new TypedMessageBuilderImpl(null, new BytesSchema());
- typedMessageBuilder.key("original key");
- typedMessageBuilder.value("original value".getBytes());
- typedMessageBuilder.eventTime(timeStamp);
- typedMessageBuilder.property(KafkaMessageRouter.PARTITION_ID, String.valueOf(partitionID));
- typedMessageBuilder.getMessage();
-
- producerInterceptors.beforeSend(null, typedMessageBuilder.getMessage());
-
- verify(mockInterceptor1, times(1)).onSend(any(ProducerRecord.class));
- verify(mockInterceptor2, times(1)).onSend(any(ProducerRecord.class));
- }
-
- @DataProvider(name = "serializers")
- public Object[][] serializers() {
- return new Object[][] {
- {
- new StringSerializer(), StringDeserializer.class
- },
- {
- new LongSerializer(), LongDeserializer.class
- },
- {
- new IntegerSerializer(), IntegerDeserializer.class,
- },
- {
- new DoubleSerializer(), DoubleDeserializer.class,
- },
- {
- new BytesSerializer(), BytesDeserializer.class
- },
- {
- new ByteBufferSerializer(), ByteBufferDeserializer.class
- },
- {
- new ByteArraySerializer(), ByteArrayDeserializer.class
- }
- };
- }
-
- @Test(dataProvider = "serializers")
- public void testGetDeserializer(Serializer serializer, Class deserializerClass) {
- Deserializer deserializer = KafkaProducerInterceptorWrapper.getDeserializer(serializer);
- assertEquals(deserializer.getClass(), deserializerClass);
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/pom.xml
deleted file mode 100644
index 4dee850..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/pom.xml
+++ /dev/null
@@ -1,125 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <kafka_0_8.version>0.8.1.1</kafka_0_8.version>
- </properties>
-
- <artifactId>pulsar-client-kafka_0_8-original</artifactId>
- <name>Pulsar Kafka compatibility 0.8 :: API (original)</name>
-
- <description>Kafka client library that publishes and consumes messages on Pulsar topics</description>
-
- <dependencies>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-common</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>testmocks</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client-admin</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>bouncy-castle-bc</artifactId>
- <version>${project.version}</version>
- <classifier>pkg</classifier>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.9.2</artifactId>
- <version>${kafka_0_8.version}</version>
- <exclusions>
- <exclusion>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerConnector.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerConnector.java
deleted file mode 100644
index 1ca0754..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerConnector.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.kafka.clients.producer.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionInitialPosition;
-import org.apache.pulsar.common.util.FutureUtil;
-import org.apache.pulsar.shade.io.netty.util.concurrent.DefaultThreadFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.TopicFilter;
-import kafka.serializer.Decoder;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * It replaces : kafka.javaapi.consumer.ConsumerConnector but not extending kafka-interface because its method has
- * KafkaStream api signature and KafkaStream is a scala class which creates unresolvable dependency conflict src:
- * https://github.com/apache/kafka/blob/0.8.2.2/core/src/main/scala/kafka/javaapi/consumer/ConsumerConnector.java
- */
-@Slf4j
-public class ConsumerConnector {
-
- private final PulsarClient client;
- private final boolean isAutoCommit;
- private final ConsumerBuilder<byte[]> consumerBuilder;
- private String clientId;
- private String groupId;
- @SuppressWarnings("rawtypes")
- private final Set<PulsarKafkaStream> topicStreams;
- private SubscriptionInitialPosition strategy = null;
- private final ScheduledExecutorService executor;
-
- public ConsumerConnector(ConsumerConfig config) {
- checkNotNull(config, "ConsumerConfig can't be null");
- clientId = config.clientId();
- groupId = config.groupId();
- isAutoCommit = config.autoCommitEnable();
- if ("largest".equalsIgnoreCase(config.autoOffsetReset())) {
- strategy = SubscriptionInitialPosition.Latest;
- } else if ("smallest".equalsIgnoreCase(config.autoOffsetReset())) {
- strategy = SubscriptionInitialPosition.Earliest;
- }
- String consumerId = !config.consumerId().isEmpty() ? config.consumerId().get() : null;
- int maxMessage = config.queuedMaxMessages();
- String serviceUrl = config.zkConnect();
-
- Properties properties = config.props() != null && config.props().props() != null ? config.props().props()
- : new Properties();
- try {
- client = PulsarClientKafkaConfig.getClientBuilder(properties).serviceUrl(serviceUrl).build();
- } catch (PulsarClientException e) {
- throw new IllegalArgumentException(
- "Failed to create pulsar-client using url = " + serviceUrl + ", properties = " + properties, e);
- }
-
- topicStreams = Sets.newConcurrentHashSet();
- consumerBuilder = client.newConsumer();
- consumerBuilder.subscriptionName(groupId);
- if (properties.containsKey("queued.max.message.chunks") && config.queuedMaxMessages() > 0) {
- consumerBuilder.receiverQueueSize(maxMessage);
- }
- if (consumerId != null) {
- consumerBuilder.consumerName(consumerId);
- }
- if (properties.containsKey("auto.commit.interval.ms") && config.autoCommitIntervalMs() > 0) {
- consumerBuilder.acknowledgmentGroupTime(config.autoCommitIntervalMs(), TimeUnit.MILLISECONDS);
- }
- this.executor = Executors.newScheduledThreadPool(1, new DefaultThreadFactory("pulsar-kafka"));
- }
-
- public <K, V> Map<String, List<PulsarKafkaStream<byte[], byte[]>>> createMessageStreams(
- Map<String, Integer> topicCountMap) {
- return createMessageStreamsByFilter(null, topicCountMap, null, null);
- }
-
- public <K, V> Map<String, List<PulsarKafkaStream<K, V>>> createMessageStreams(Map<String, Integer> topicCountMap,
- Decoder<K> keyDecoder, Decoder<V> valueDecoder) {
- return createMessageStreamsByFilter(null, topicCountMap, keyDecoder, valueDecoder);
- }
-
- public <K, V> Map<String, List<PulsarKafkaStream<K, V>>> createMessageStreamsByFilter(TopicFilter topicFilter,
- Map<String, Integer> topicCountMap, Decoder<K> keyDecoder, Decoder<V> valueDecoder) {
-
- Map<String, List<PulsarKafkaStream<K, V>>> streams = Maps.newHashMap();
-
- topicCountMap.forEach((topic, count) -> {
- try {
- Consumer<byte[]> consumer = consumerBuilder.topic(topic).subscribe();
- resetOffsets(consumer, strategy);
- log.info("Creating stream for {}-{} with config {}", topic, groupId, consumerBuilder.toString());
- for (int i = 0; i < count; i++) {
- PulsarKafkaStream<K, V> stream = new PulsarKafkaStream<>(keyDecoder, valueDecoder, consumer,
- isAutoCommit, clientId);
- // if multiple thread-count present then client expects multiple streams reading from the same
- // topic. so, create multiple stream using the same consumer
- streams.computeIfAbsent(topic, key -> Lists.newArrayList()).add(stream);
- topicStreams.add(stream);
- }
- } catch (PulsarClientException e) {
- log.error("Failed to subscribe on topic {} with group-id {}, {}", topic, groupId, e.getMessage(), e);
- throw new RuntimeException("Failed to subscribe on topic " + topic, e);
- }
- });
- return streams;
- }
-
- public List<PulsarKafkaStream<byte[], byte[]>> createMessageStreamsByFilter(TopicFilter topicFilter) {
- throw new UnsupportedOperationException("method not supported");
- }
-
- public List<PulsarKafkaStream<byte[], byte[]>> createMessageStreamsByFilter(TopicFilter topicFilter, int arg1) {
- throw new UnsupportedOperationException("method not supported");
- }
-
- public <K, V> List<PulsarKafkaStream<K, V>> createMessageStreamsByFilter(TopicFilter topicFilter, int arg1,
- Decoder<K> keyDecoder, Decoder<V> valueDecoder) {
- throw new UnsupportedOperationException("method not supported");
- }
-
- @SuppressWarnings("unchecked")
- public List<CompletableFuture<Void>> commitOffsetsAsync() {
- return topicStreams.stream().map(stream -> (CompletableFuture<Void>) stream.commitOffsets())
- .collect(Collectors.toList());
- }
-
- /**
- * Commit the offsets of all broker partitions connected by this connector.
- */
- public void commitOffsets() {
- commitOffsetsAsync();
- }
-
- public void commitOffsets(boolean retryOnFailure) {
- FutureUtil.waitForAll(commitOffsetsAsync()).handle((res, ex) -> {
- if (ex != null) {
- if (log.isDebugEnabled()) {
- log.debug("Failed to commit offset {}, retrying {}", ex.getMessage(), retryOnFailure);
- }
- if (retryOnFailure) {
- this.executor.schedule(() -> commitOffsets(retryOnFailure), 30, TimeUnit.SECONDS);
- }
- }
- return null;
- });
- }
-
- /**
- * Shut down the connector
- */
- public void shutdown() {
- if (executor != null) {
- executor.shutdown();
- }
- if (topicStreams != null) {
- topicStreams.forEach(stream -> {
- try {
- stream.close();
- } catch (Exception e) {
- log.warn("Failed to close stream {}, {}", stream, e.getMessage());
- }
- });
- }
- try {
- client.close();
- } catch (PulsarClientException e) {
- log.warn("Failed to close client {}", e.getMessage());
- }
- }
-
- private void resetOffsets(Consumer<byte[]> consumer, SubscriptionInitialPosition strategy) {
- if (strategy == null) {
- return;
- }
- log.info("Resetting partition {} for group-id {} and seeking to {} position", consumer.getTopic(),
- consumer.getSubscription(), strategy);
- try {
- if (strategy == SubscriptionInitialPosition.Earliest) {
- consumer.seek(MessageId.earliest);
- } else {
- consumer.seek(MessageId.latest);
- }
- } catch (PulsarClientException e) {
- log.warn("Failed to reset offset for consumer {} to {}, {}", consumer.getTopic(), strategy,
- e.getMessage(), e);
- }
- }
-
- @VisibleForTesting
- public ConsumerBuilder<byte[]> getConsumerBuilder() {
- return consumerBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerIterator.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerIterator.java
deleted file mode 100644
index 926e765..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/ConsumerIterator.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import java.util.Base64;
-import java.util.Iterator;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang3.SerializationUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.apache.pulsar.common.naming.TopicName;
-
-import kafka.serializer.Decoder;
-import kafka.serializer.StringDecoder;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class ConsumerIterator<K, V> implements Iterator<PulsarMessageAndMetadata<K, V>>, AutoCloseable {
-
- private final Consumer<byte[]> consumer;
- private final ConcurrentLinkedQueue<Message<byte[]>> receivedMessages;
- private final Optional<Decoder<K>> keyDeSerializer;
- private final Optional<Decoder<V>> valueDeSerializer;
- private final boolean isAutoCommit;
- private volatile MessageId lastConsumedMessageId;
- private static final kafka.serializer.DefaultDecoder DEFAULT_DECODER = new kafka.serializer.DefaultDecoder(null);
-
- public ConsumerIterator(Consumer<byte[]> consumer, ConcurrentLinkedQueue<Message<byte[]>> receivedMessages,
- Optional<Decoder<K>> keyDeSerializer, Optional<Decoder<V>> valueDeSerializer, boolean isAutoCommit) {
- this.consumer = consumer;
- this.receivedMessages = receivedMessages;
- this.keyDeSerializer = keyDeSerializer;
- this.valueDeSerializer = valueDeSerializer;
- this.isAutoCommit = isAutoCommit;
- }
-
- @Override
- public boolean hasNext() {
- try {
- Message<byte[]> msg = consumer.receive(10, TimeUnit.MILLISECONDS);
- if (msg != null) {
- receivedMessages.offer(msg);
- return true;
- }
- } catch (PulsarClientException e) {
- if (log.isDebugEnabled()) {
- log.debug("Failed to receive message for {}-{}, {}", consumer.getTopic(), consumer.getSubscription(),
- e.getMessage());
- }
- }
- return false;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public PulsarMessageAndMetadata<K, V> next() {
-
- Message<byte[]> msg = receivedMessages.poll();
- if (msg == null) {
- try {
- msg = consumer.receive();
- } catch (PulsarClientException e) {
- log.warn("Failed to receive message for {}-{}, {}", consumer.getTopic(), consumer.getSubscription(),
- e.getMessage(), e);
- throw new RuntimeException(
- "failed to receive message from " + consumer.getTopic() + "-" + consumer.getSubscription());
- }
- }
-
- int partition = TopicName.getPartitionIndex(consumer.getTopic());
- long offset = MessageIdUtils.getOffset(msg.getMessageId());
- String key = msg.getKey();
- byte[] value = msg.getValue();
-
- K desKey = null;
- V desValue = null;
-
- if (StringUtils.isNotBlank(key)) {
- if (keyDeSerializer.isPresent() && keyDeSerializer.get() instanceof StringDecoder) {
- desKey = (K) key;
- } else {
- byte[] decodedBytes = Base64.getDecoder().decode(key);
- desKey = keyDeSerializer.isPresent() ? keyDeSerializer.get().fromBytes(decodedBytes)
- : (K) DEFAULT_DECODER.fromBytes(decodedBytes);
- }
- }
-
- if (value != null) {
- desValue = valueDeSerializer.isPresent() ? valueDeSerializer.get().fromBytes(msg.getData())
- : (V) DEFAULT_DECODER.fromBytes(msg.getData());
- }
-
- PulsarMessageAndMetadata<K, V> msgAndMetadata = new PulsarMessageAndMetadata<>(consumer.getTopic(), partition,
- null, offset, keyDeSerializer.orElse(null), valueDeSerializer.orElse(null), desKey, desValue);
-
- if (isAutoCommit) {
- // Commit the offset of previously dequeued messages
- consumer.acknowledgeCumulativeAsync(msg);
- }
-
- lastConsumedMessageId = msg.getMessageId();
- return msgAndMetadata;
- }
-
- protected CompletableFuture<Void> commitOffsets() {
- MessageId msgId = lastConsumedMessageId;
- if (msgId != null) {
- return this.consumer.acknowledgeCumulativeAsync(msgId);
- }
- return CompletableFuture.completedFuture(null);
- }
-
- @Override
- public void close() throws Exception {
- if (consumer != null) {
- consumer.close();
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarConsumerConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarConsumerConfig.java
deleted file mode 100644
index 8aa0359..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarConsumerConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import java.util.Properties;
-
-public class PulsarConsumerConfig extends kafka.consumer.ConsumerConfig {
-
- public PulsarConsumerConfig(Properties originalProps) {
- super(originalProps);
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
deleted file mode 100644
index cdd62a4..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-/**
- * This class replaces "kafka.consumer.Consumer" and not extending kafka-interface as it's a final scala class.
- *
- */
-public class PulsarKafkaConsumer {
-
- public static ConsumerConnector createJavaConsumerConnector(PulsarConsumerConfig config) {
- return new ConsumerConnector(config);
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaStream.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaStream.java
deleted file mode 100644
index 3a1231d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaStream.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.Message;
-
-import com.google.common.collect.Queues;
-
-import kafka.serializer.Decoder;
-
-/**
- * We can't extends "kafka.consumer.KafkaStream<K,V>" because it's a scala class which brings ambiguous overriden
- * methods that gives compilation errors.
- *
- * @param <K>
- * @param <V>
- */
-public class PulsarKafkaStream<K, V> implements Iterable<PulsarMessageAndMetadata<K, V>>, AutoCloseable {
-
- private final Optional<Decoder<K>> keyDeSerializer;
- private final Optional<Decoder<V>> valueDeSerializer;
- private final ConsumerIterator<K, V> iterator;
-
- private final ConcurrentLinkedQueue<Message<byte[]>> receivedMessages = Queues.newConcurrentLinkedQueue();
-
- public PulsarKafkaStream(Decoder<K> keyDecoder, Decoder<V> valueDecoder, Consumer<byte[]> consumer,
- boolean isAutoCommit, String clientId) {
- this.keyDeSerializer = Optional.ofNullable(keyDecoder);
- this.valueDeSerializer = Optional.ofNullable(valueDecoder);
- this.iterator = new ConsumerIterator<>(consumer, receivedMessages, keyDeSerializer, valueDeSerializer,
- isAutoCommit);
- }
-
- @Override
- public ConsumerIterator<K, V> iterator() {
- return iterator;
- }
-
- public CompletableFuture<Void> commitOffsets() {
- return iterator.commitOffsets();
- }
-
- @Override
- public void close() throws Exception {
- iterator.close();
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarMessageAndMetadata.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarMessageAndMetadata.java
deleted file mode 100644
index 2c76501..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/consumer/PulsarMessageAndMetadata.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import kafka.message.Message;
-import kafka.message.MessageAndMetadata;
-import kafka.serializer.Decoder;
-
-public class PulsarMessageAndMetadata<K, V> extends MessageAndMetadata<K, V> {
-
- private static final long serialVersionUID = 1L;
- private final String topic;
- private final int partition;
- private final long offset;
- private final K key;
- private final V value;
- private final Decoder<K> keyDecoder;
- private final Decoder<V> valueDecoder;
-
- public PulsarMessageAndMetadata(String topic, int partition, Message rawMessage, long offset, Decoder<K> keyDecoder,
- Decoder<V> valueDecoder, K key, V value) {
- super(topic, partition, rawMessage, offset, keyDecoder, valueDecoder);
- this.topic = topic;
- this.partition = partition;
- this.offset = offset;
- this.keyDecoder = keyDecoder;
- this.valueDecoder = valueDecoder;
- this.key = key;
- this.value = value;
- }
-
- @Override
- public String topic() {
- return topic;
- }
-
- @Override
- public int productArity() {
- return 0;
- }
-
- @Override
- public int partition() {
- return partition;
- }
-
- @Override
- public long offset() {
- return offset;
- }
-
- @Override
- public V message() {
- return this.value;
- }
-
- @Override
- public K key() {
- return key;
- }
-
- @Override
- public Decoder<V> valueDecoder() {
- return this.valueDecoder;
- }
-
- @Override
- public Decoder<K> keyDecoder() {
- return this.keyDecoder;
- }
-
- @Override
- public Message rawMessage$1() {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarClientKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarClientKafkaConfig.java
deleted file mode 100644
index c71d7a7..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarClientKafkaConfig.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminBuilder;
-import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-
-public class PulsarClientKafkaConfig {
-
- /// Config variables
- public static final String AUTHENTICATION_CLASS = "pulsar.authentication.class";
- public static final String AUTHENTICATION_PARAMS_MAP = "pulsar.authentication.params.map";
- public static final String AUTHENTICATION_PARAMS_STRING = "pulsar.authentication.params.string";
- public static final String USE_TLS = "pulsar.use.tls";
- public static final String TLS_TRUST_CERTS_FILE_PATH = "pulsar.tls.trust.certs.file.path";
- public static final String TLS_ALLOW_INSECURE_CONNECTION = "pulsar.tls.allow.insecure.connection";
- public static final String TLS_HOSTNAME_VERIFICATION = "pulsar.tls.hostname.verification";
-
- public static final String OPERATION_TIMEOUT_MS = "pulsar.operation.timeout.ms";
- public static final String STATS_INTERVAL_SECONDS = "pulsar.stats.interval.seconds";
- public static final String NUM_IO_THREADS = "pulsar.num.io.threads";
-
- public static final String CONNECTIONS_PER_BROKER = "pulsar.connections.per.broker";
- public static final String ADMIN_SERVICE_URL = "pulsar.admin.service.url";
-
- public static final String USE_TCP_NODELAY = "pulsar.use.tcp.nodelay";
-
- public static final String CONCURRENT_LOOKUP_REQUESTS = "pulsar.concurrent.lookup.requests";
- public static final String MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION = "pulsar.max.number.rejected.request.per.connection";
-
- public static ClientBuilder getClientBuilder(Properties properties) {
- ClientBuilder clientBuilder = PulsarClient.builder();
- if (properties == null) {
- return clientBuilder;
- }
-
- if (properties.containsKey(AUTHENTICATION_CLASS)) {
- String className = properties.getProperty(AUTHENTICATION_CLASS);
- try {
- if (properties.containsKey(AUTHENTICATION_PARAMS_STRING)) {
- String authParamsString = (String) properties.get(AUTHENTICATION_PARAMS_STRING);
- clientBuilder.authentication(className, authParamsString);
- } else if (properties.containsKey(AUTHENTICATION_PARAMS_MAP)) {
- Map<String, String> authParams = (Map<String, String>) properties.get(AUTHENTICATION_PARAMS_MAP);
- clientBuilder.authentication(className, authParams);
- } else {
- @SuppressWarnings("unchecked")
- Class<Authentication> clazz = (Class<Authentication>) Class.forName(className);
- Authentication auth = clazz.newInstance();
- clientBuilder.authentication(auth);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- clientBuilder.enableTls(Boolean.parseBoolean(properties.getProperty(USE_TLS, "false")));
- clientBuilder.allowTlsInsecureConnection(
- Boolean.parseBoolean(properties.getProperty(TLS_ALLOW_INSECURE_CONNECTION, "false")));
- clientBuilder.enableTlsHostnameVerification(
- Boolean.parseBoolean(properties.getProperty(TLS_HOSTNAME_VERIFICATION, "false")));
-
- if (properties.containsKey(TLS_TRUST_CERTS_FILE_PATH)) {
- clientBuilder.tlsTrustCertsFilePath(properties.getProperty(TLS_TRUST_CERTS_FILE_PATH));
- }
-
- if (properties.containsKey(OPERATION_TIMEOUT_MS)) {
- clientBuilder.operationTimeout(Integer.parseInt(properties.getProperty(OPERATION_TIMEOUT_MS)),
- TimeUnit.MILLISECONDS);
- }
-
- if (properties.containsKey(STATS_INTERVAL_SECONDS)) {
- clientBuilder.statsInterval(Integer.parseInt(properties.getProperty(STATS_INTERVAL_SECONDS)),
- TimeUnit.SECONDS);
- }
-
- if (properties.containsKey(NUM_IO_THREADS)) {
- clientBuilder.ioThreads(Integer.parseInt(properties.getProperty(NUM_IO_THREADS)));
- }
-
- if (properties.containsKey(CONNECTIONS_PER_BROKER)) {
- clientBuilder.connectionsPerBroker(Integer.parseInt(properties.getProperty(CONNECTIONS_PER_BROKER)));
- }
-
- if (properties.containsKey(USE_TCP_NODELAY)) {
- clientBuilder.enableTcpNoDelay(Boolean.parseBoolean(properties.getProperty(USE_TCP_NODELAY)));
- }
-
- if (properties.containsKey(CONCURRENT_LOOKUP_REQUESTS)) {
- clientBuilder
- .maxConcurrentLookupRequests(Integer.parseInt(properties.getProperty(CONCURRENT_LOOKUP_REQUESTS)));
- }
-
- if (properties.containsKey(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)) {
- clientBuilder.maxNumberOfRejectedRequestPerConnection(
- Integer.parseInt(properties.getProperty(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)));
- }
-
- return clientBuilder;
- }
-
- public static PulsarAdminBuilder getAdminBuilder(String serviceUrl, Properties properties) {
- PulsarAdminBuilder adminBuilder = PulsarAdmin.builder();
- if (properties == null) {
- return adminBuilder;
- }
-
- adminBuilder.serviceHttpUrl(properties.getProperty(ADMIN_SERVICE_URL, serviceUrl));
-
- if (properties.containsKey(AUTHENTICATION_CLASS)) {
- String className = properties.getProperty(AUTHENTICATION_CLASS);
- try {
- if (properties.containsKey(AUTHENTICATION_PARAMS_STRING)) {
- String authParamsString = (String) properties.get(AUTHENTICATION_PARAMS_STRING);
- adminBuilder.authentication(className, authParamsString);
- } else if (properties.containsKey(AUTHENTICATION_PARAMS_MAP)) {
- Map<String, String> authParams = (Map<String, String>) properties.get(AUTHENTICATION_PARAMS_MAP);
- adminBuilder.authentication(className, authParams);
- } else {
- @SuppressWarnings("unchecked")
- Class<Authentication> clazz = (Class<Authentication>) Class.forName(className);
- Authentication auth = clazz.newInstance();
- adminBuilder.authentication(auth);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- adminBuilder.allowTlsInsecureConnection(
- Boolean.parseBoolean(properties.getProperty(TLS_ALLOW_INSECURE_CONNECTION, "false")));
- adminBuilder.enableTlsHostnameVerification(
- Boolean.parseBoolean(properties.getProperty(TLS_HOSTNAME_VERIFICATION, "false")));
-
- if (properties.containsKey(TLS_TRUST_CERTS_FILE_PATH)) {
- adminBuilder.tlsTrustCertsFilePath(properties.getProperty(TLS_TRUST_CERTS_FILE_PATH));
- }
-
- if (properties.containsKey(OPERATION_TIMEOUT_MS)) {
- adminBuilder.connectionTimeout(Integer.parseInt(properties.getProperty(OPERATION_TIMEOUT_MS)),
- TimeUnit.MILLISECONDS);
- }
-
- return adminBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
deleted file mode 100644
index 7d6e146..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.Serializable;
-import java.lang.reflect.Constructor;
-import java.util.Base64;
-import java.util.List;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang3.SerializationUtils;
-import org.apache.pulsar.client.api.CompressionType;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageRouter;
-import org.apache.pulsar.client.api.MessageRoutingMode;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.TopicMetadata;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-
-import com.google.common.annotations.VisibleForTesting;
-
-//Questions
-/**
- * 1. What's the auth method
- * 2. What if message publish fails with async
- */
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.DefaultPartitioner;
-import kafka.producer.KeyedMessage;
-import kafka.producer.Partitioner;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.Encoder;
-import kafka.serializer.StringEncoder;
-import kafka.utils.VerifiableProperties;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class PulsarKafkaProducer<K, V> extends Producer<K, V> {
-
- private final PulsarClient client;
- private final ProducerBuilder<byte[]> pulsarProducerBuilder;
- private final Partitioner partitioner;
- private final Encoder<K> keySerializer;
- private final Encoder<V> valueSerializer;
- private final boolean isSendAsync;
-
- public static String KAFKA_KEY_MAX_QUEUE_BUFFERING_TIME_MS = "queue.buffering.max.ms";
- public static String KAFKA_KEY_MAX_QUEUE_BUFFERING_MESSAGES = "queue.buffering.max.messages";
- public static String KAFKA_KEY_MAX_BATCH_MESSAGES = "batch.num.messages";
- public static String KAFKA_KEY_REQUEST_TIMEOUT_MS = "request.timeout.ms";
-
- private final ConcurrentMap<String, org.apache.pulsar.client.api.Producer<byte[]>> producers = new ConcurrentHashMap<>();
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public PulsarKafkaProducer(ProducerConfig config) {
- super((kafka.producer.Producer) null);
- partitioner = config.partitionerClass() != null
- ? newInstance(config.partitionerClass(), Partitioner.class, config.props())
- : new DefaultPartitioner(config.props());
- // kafka-config returns default serializer if client doesn't configure it
- checkNotNull(config.keySerializerClass(), "key-serializer class can't be null");
- checkNotNull(config.serializerClass(), "value-serializer class can't be null");
- keySerializer = newInstance(config.keySerializerClass(), Encoder.class, config.props());
- valueSerializer = newInstance(config.serializerClass(), Encoder.class, config.props());
-
- Properties properties = config.props() != null && config.props().props() != null ? config.props().props()
- : new Properties();
- String serviceUrl = config.brokerList();
- try {
- client = PulsarClientKafkaConfig.getClientBuilder(properties).serviceUrl(serviceUrl).build();
- } catch (PulsarClientException e) {
- throw new IllegalArgumentException(
- "Failed to create pulsar-client using url = " + serviceUrl + ", properties = " + properties, e);
- }
- pulsarProducerBuilder = client.newProducer();
-
- // doc: https://kafka.apache.org/08/documentation.html#producerapi
- // api-doc:
- // https://github.com/apache/kafka/blob/0.8.2.2/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
-
- // queue.enqueue.timeout.ms: The amount of time to block before dropping messages when running in async mode and
- // the buffer has reached queue.buffering.max.messages. If set to 0 events will be enqueued immediately or
- // dropped if the queue is full (the producer send call will never block). If set to -1 the producer will block
- // indefinitely and never willingly drop a send.
- boolean blockIfQueueFull = config.queueEnqueueTimeoutMs() == -1 ? true : false;
- // This parameter specifies whether the messages are sent asynchronously in a background thread. Valid values
- // are (1) async for asynchronous send and (2) sync for synchronous send. By setting the producer to async we
- // allow batching together of requests (which is great for throughput) but open the possibility of a failure of
- // the client machine dropping unsent data.
- isSendAsync = "async".equalsIgnoreCase(config.producerType());
- CompressionType compressionType = CompressionType.NONE;
- // Valid values are "none", "gzip" and "snappy".
- if ("gzip".equals(config.compressionCodec().name())) {
- compressionType = CompressionType.ZLIB;
- } else if ("snappy".equals(config.compressionCodec().name())) {
- compressionType = CompressionType.SNAPPY;
- }
- long batchDelayMs = config.queueBufferingMaxMs();
-
- if (properties.containsKey(KAFKA_KEY_MAX_QUEUE_BUFFERING_MESSAGES)) {
- pulsarProducerBuilder.maxPendingMessages(config.queueBufferingMaxMessages());
- }
- if (properties.containsKey(KAFKA_KEY_MAX_BATCH_MESSAGES)) {
- pulsarProducerBuilder.batchingMaxMessages(config.batchNumMessages());
- }
- if (properties.containsKey(KAFKA_KEY_MAX_QUEUE_BUFFERING_TIME_MS)) {
- pulsarProducerBuilder.batchingMaxPublishDelay(batchDelayMs, TimeUnit.MILLISECONDS);
- }
- if (properties.containsKey(KAFKA_KEY_REQUEST_TIMEOUT_MS)) {
- pulsarProducerBuilder.sendTimeout(config.requestTimeoutMs(), TimeUnit.MILLISECONDS);
- }
-
- pulsarProducerBuilder.blockIfQueueFull(blockIfQueueFull).compressionType(compressionType);
-
- }
-
- public PulsarKafkaProducer(kafka.producer.Producer<K, V> producer) {
- this(producer.config());
- }
-
- @Override
- public void send(KeyedMessage<K, V> message) {
- org.apache.pulsar.client.api.Producer<byte[]> producer;
-
- try {
- producer = producers.computeIfAbsent(message.topic(), topic -> createNewProducer(topic));
- } catch (Exception e) {
- throw new IllegalArgumentException("Failed to create producer for " + message.topic(), e);
- }
-
- TypedMessageBuilder<byte[]> messageBuilder = producer.newMessage();
- buildMessage(messageBuilder, message);
-
- if (isSendAsync) {
- // what if message publish fails:
- // according to : https://kafka.apache.org/08/documentation.html#producerapi
- // async: opens the possibility of a failure of the client machine dropping unsent data
- messageBuilder.sendAsync().handle((res, ex) -> {
- if (ex != null) {
- log.warn("publish failed for {}", producer.getTopic(), ex);
- }
- return null;
- });
- } else {
- try {
- messageBuilder.send();
- } catch (PulsarClientException e) {
- log.warn("publish failed for {}", producer.getTopic(), e);
- throw new IllegalStateException("Failed to publish message " + message.topic(), e);
- }
- }
-
- }
-
- @Override
- public void send(List<KeyedMessage<K, V>> messages) {
- if (messages != null) {
- messages.forEach(this::send);
- }
- }
-
- private void buildMessage(TypedMessageBuilder<byte[]> builder, KeyedMessage<K, V> message) {
- if (message.key() != null) {
- String key = getKey(message.topic(), message.key());
- builder.key(key);
- }
-
- byte[] value = valueSerializer.toBytes(message.message());
- builder.value(value);
- }
-
- private String getKey(String topic, K key) {
- // If key is a String, we can use it as it is, otherwise, serialize to byte[] and encode in base64
- if (keySerializer!=null && keySerializer instanceof StringEncoder) {
- return (String) key;
- } else {
- byte[] keyBytes = keySerializer.toBytes(key);
- return Base64.getEncoder().encodeToString(keyBytes);
- }
- }
-
- private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
- try {
- pulsarProducerBuilder.messageRoutingMode(MessageRoutingMode.CustomPartition);
- pulsarProducerBuilder.messageRouter(new MessageRouter() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public int choosePartition(Message<?> msg, TopicMetadata metadata) {
- // https://kafka.apache.org/08/documentation.html#producerapi
- // The default partitioner is based on the hash of the key.
- return partitioner.partition(msg.getKey(), metadata.numPartitions());
- }
- });
- log.info("Creating producer for topic {} with config {}", topic, pulsarProducerBuilder.toString());
- return pulsarProducerBuilder.clone().topic(topic).create();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- public <T> T newInstance(String key, Class<T> t, VerifiableProperties properties) {
- Class<?> c = null;
- try {
- c = Class.forName(key);
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
- throw new IllegalArgumentException("class not found for :" + key);
- }
- if (c == null)
- return null;
- Object o = newInstance(c, properties);
- if (!t.isInstance(o)) {
- throw new IllegalArgumentException(c.getName() + " is not an instance of " + t.getName());
- }
- return t.cast(o);
- }
-
- public static <T> T newInstance(Class<T> c, VerifiableProperties properties) {
- try {
- try {
- Constructor<T> constructor = c.getConstructor(VerifiableProperties.class);
- constructor.setAccessible(true);
- return constructor.newInstance(properties);
- } catch (Exception e) {
- // Ok.. not a default implementation class
- }
- return c.newInstance();
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException("Could not instantiate class " + c.getName(), e);
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(
- "Could not instantiate class " + c.getName() + " Does it have a public no-argument constructor?",
- e);
- } catch (NullPointerException e) {
- throw new IllegalArgumentException("Requested class was null", e);
- }
- }
-
- @Override
- public void close() {
- if (producers != null) {
- producers.forEach((topic, producer) -> {
- try {
- producer.close();
- } catch (PulsarClientException e) {
- log.warn("Failed to close producer for {}: {}", topic, e.getMessage());
- }
- });
- }
- if (client != null) {
- try {
- client.close();
- } catch (PulsarClientException e) {
- log.warn("Failed to close pulsar-client", e);
- }
- }
- }
-
- @VisibleForTesting
- public ProducerBuilder<byte[]> getPulsarProducerBuilder() {
- return pulsarProducerBuilder;
- }
-
- @VisibleForTesting
- public Partitioner getPartitioner() {
- return partitioner;
- }
-
- @VisibleForTesting
- public Encoder<K> getKeySerializer() {
- return keySerializer;
- }
-
- @VisibleForTesting
- public Encoder<V> getValueSerializer() {
- return valueSerializer;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarByteBufferMessageSet.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarByteBufferMessageSet.java
deleted file mode 100644
index 68ee247..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarByteBufferMessageSet.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.Reader;
-
-import com.google.common.collect.Queues;
-
-import kafka.message.MessageAndOffset;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class PulsarByteBufferMessageSet extends kafka.javaapi.message.ByteBufferMessageSet {
- private final MessageAndOffsetIterator iterator;
-
- public PulsarByteBufferMessageSet(Reader<byte[]> reader) {
- super(Collections.emptyList());
- this.iterator = new MessageAndOffsetIterator(reader);
- }
-
- @Override
- public Iterator<MessageAndOffset> iterator() {
- return iterator;
- }
-
- public static class MessageAndOffsetIterator implements Iterator<MessageAndOffset> {
-
- private final Reader<byte[]> reader;
- private final ConcurrentLinkedQueue<org.apache.pulsar.client.api.Message<byte[]>> receivedMessages;
-
- public MessageAndOffsetIterator(Reader<byte[]> reader) {
- this.reader = reader;
- this.receivedMessages = Queues.newConcurrentLinkedQueue();
- }
-
- @Override
- public boolean hasNext() {
- try {
- org.apache.pulsar.client.api.Message<byte[]> msg = reader.readNext(10, TimeUnit.MILLISECONDS);
- if (msg != null) {
- receivedMessages.offer(msg);
- return true;
- }
- } catch (PulsarClientException e) {
- if (log.isDebugEnabled()) {
- log.debug("Failed to receive message for {}, {}", reader.getTopic(), e.getMessage());
- }
- }
- return false;
- }
-
- @Override
- public PulsarMsgAndOffset next() {
-
- org.apache.pulsar.client.api.Message<byte[]> msg = receivedMessages.poll();
- if (msg == null) {
- try {
- msg = reader.readNext();
- } catch (PulsarClientException e) {
- log.warn("Failed to receive message for {}, {}", reader.getTopic(), e.getMessage(), e);
- throw new RuntimeException("failed to receive message from " + reader.getTopic());
- }
- }
-
- String key = msg.getKey();
- byte[] value = msg.getValue();
-
- return new PulsarMsgAndOffset(new PulsarMessage(key, value), msg.getMessageId());
- }
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarFetchResponse.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarFetchResponse.java
deleted file mode 100644
index 6f47e8b..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarFetchResponse.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Map;
-
-import org.apache.pulsar.client.api.Reader;
-
-import kafka.javaapi.FetchResponse;
-
-public class PulsarFetchResponse extends FetchResponse {
-
- private final Map<String, Reader<byte[]>> topicReaderMap;
- private boolean hasError = false;
-
- public PulsarFetchResponse(Map<String, Reader<byte[]>> topicConsumerMap, boolean hasError) {
- super(null);
- this.topicReaderMap = topicConsumerMap;
- this.hasError = hasError;
- }
-
- @Override
- public boolean hasError() {
- return this.hasError;
- }
-
- @Override
- public PulsarByteBufferMessageSet messageSet(String topic, int partition) {
- return new PulsarByteBufferMessageSet(topicReaderMap.get(topic));
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarKafkaSimpleConsumer.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarKafkaSimpleConsumer.java
deleted file mode 100644
index eb34d77..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarKafkaSimpleConsumer.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kafka.clients.producer.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.Reader;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
-import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats;
-
-import com.google.common.collect.Maps;
-
-import kafka.api.FetchRequest;
-import kafka.api.PartitionFetchInfo;
-import kafka.api.PartitionOffsetRequestInfo;
-import kafka.common.ErrorMapping;
-import kafka.common.OffsetMetadataAndError;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetCommitResponse;
-import kafka.javaapi.TopicMetadataRequest;
-import kafka.javaapi.consumer.SimpleConsumer;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Note: <br/>
- * - SimpleConsumer doesn't work well with pulsar-batch messages because client-app uses raw-offset to commit offset for
- * a given group-id. <br/>
- * - In order to work with partitioned-topic and batch messages: use custom api
- * {@link #PulsarMsgAndOffset::getFullOffset()} to fetch offset with {@link MessageId} and commit offset with the same
- * message-Id {@link PulsarOffsetMetadataAndError::PulsarOffsetMetadataAndError(messageId..)}.
- *
- */
-@Slf4j
-public class PulsarKafkaSimpleConsumer extends SimpleConsumer {
-
- private final String host;
- private final int port;
- private final String clientId;
- private final PulsarClient client;
- private final PulsarAdmin admin;
- private final Map<TopicGroup, Consumer<byte[]>> topicConsumerMap;
- private final SubscriptionType subscriptionType;
- public static final String SUBSCRIPTION_TYPE = "pulsar.subscription.type";
- public static final String HTTP_SERVICE_URL = "pulsar.http.service.url";
-
- public PulsarKafkaSimpleConsumer(String host, int port, int soTimeout, int bufferSize, String clientId) {
- this(host, port, soTimeout, bufferSize, clientId, new Properties());
- }
-
- /**
- *
- * @param host
- * pulsar-broker service url
- * @param port
- * n/a
- * @param soTimeout
- * n/a
- * @param bufferSize
- * n/a
- * @param clientId
- * client-id
- * @param properties
- * properties to retrieve authentication params by {@link PulsarClientKafkaConfig}
- */
- public PulsarKafkaSimpleConsumer(String host, int port, int soTimeout, int bufferSize, String clientId,
- Properties properties) {
- super(host, port, soTimeout, bufferSize, clientId);
- this.host = host;
- this.port = port;
- this.clientId = clientId;
- try {
- client = PulsarClientKafkaConfig.getClientBuilder(properties).serviceUrl(host).build();
- } catch (PulsarClientException e) {
- log.warn("Failed to create pulsar client for {} and properties {}", host, properties);
- throw new RuntimeException("Failed to create pulsar client " + host, e);
- }
- try {
- String url = properties.getProperty(HTTP_SERVICE_URL, host);
- admin = PulsarClientKafkaConfig.getAdminBuilder(url, properties).build();
- } catch (PulsarClientException e) {
- log.warn("Failed to create pulsar admin for {} and properties {}", host, properties);
- throw new RuntimeException("Failed to create pulsar admin " + host, e);
- }
- this.topicConsumerMap = new ConcurrentHashMap<>(8, 0.75f, 1);
- this.subscriptionType = getSubscriptionType(properties);
- }
-
- @Override
- public PulsarFetchResponse fetch(FetchRequest request) {
- try {
- Map<String, Reader<byte[]>> topicReaderMap = createTopicReaders(request);
- return new PulsarFetchResponse(topicReaderMap, false);
- } catch (Exception e) {
- log.warn("Failed to process fetch request{}, {}", request, e.getMessage());
- return new PulsarFetchResponse(null, true);
- }
- }
-
- private Map<String, Reader<byte[]>> createTopicReaders(FetchRequest request) {
- Map<String, Reader<byte[]>> topicReaderMap = Maps.newHashMap();
- scala.collection.immutable.Map<String, scala.collection.immutable.Map<TopicAndPartition, PartitionFetchInfo>> reqInfo = request
- .requestInfoGroupedByTopic();
- Map<String, scala.collection.immutable.Map<TopicAndPartition, PartitionFetchInfo>> topicPartitionMap = scala.collection.JavaConverters
- .mapAsJavaMapConverter(reqInfo).asJava();
- for (Entry<String, scala.collection.immutable.Map<TopicAndPartition, PartitionFetchInfo>> topicPartition : topicPartitionMap
- .entrySet()) {
- final String topicName = topicPartition.getKey();
- Map<TopicAndPartition, PartitionFetchInfo> topicOffsetMap = scala.collection.JavaConverters
- .mapAsJavaMapConverter(topicPartition.getValue()).asJava();
- if (topicOffsetMap != null && !topicOffsetMap.isEmpty()) {
- // pulsar-kafka adapter doesn't deal with partition so, assuming only 1 topic-metadata per topic name
- Entry<TopicAndPartition, PartitionFetchInfo> topicOffset = topicOffsetMap.entrySet().iterator().next();
- long offset = topicOffset.getValue().offset();
- String topic = getTopicName(topicOffset.getKey());
- MessageId msgId = getMessageId(offset);
- try {
- Reader<byte[]> reader = client.newReader().readerName(clientId).topic(topic).startMessageId(msgId)
- .create();
- log.info("Successfully created reader for {} at msg-id {}", topic, msgId);
- topicReaderMap.put(topicName, reader);
- } catch (PulsarClientException e) {
- log.warn("Failed to create reader for topic {}", topic, e);
- throw new RuntimeException("Failed to create reader for " + topic, e);
- }
- }
- }
- return topicReaderMap;
- }
-
- private MessageId getMessageId(long offset) {
- if (kafka.api.OffsetRequest.EarliestTime() == offset) {
- return MessageId.earliest;
- } else if (kafka.api.OffsetRequest.LatestTime() == offset) {
- return MessageId.latest;
- } else {
- return MessageIdUtils.getMessageId(offset);
- }
- }
-
- @Override
- public PulsarTopicMetadataResponse send(TopicMetadataRequest request) {
- List<String> topics = request.topics();
- PulsarTopicMetadataResponse response = new PulsarTopicMetadataResponse(admin, host, port, topics);
- return response;
- }
-
- // It's @Overriden method of: OffsetResponse getOffsetsBefore(OffsetRequest or)
- public PulsarOffsetResponse getOffsetsBefore(PulsarOffsetRequest or) {
- Map<TopicAndPartition, PartitionOffsetRequestInfo> request = or.getRequestInfo();
- Map<TopicAndPartition, Long> offsetResoponse = Maps.newHashMap();
- for (Entry<TopicAndPartition, PartitionOffsetRequestInfo> topicPartitionRequest : request.entrySet()) {
- TopicAndPartition topic = topicPartitionRequest.getKey();
- long time = topicPartitionRequest.getValue().time();
- if (time != kafka.api.OffsetRequest.EarliestTime() && time != kafka.api.OffsetRequest.LatestTime()) {
- throw new IllegalArgumentException("Time has to be from EarliestTime or LatestTime");
- }
- offsetResoponse.put(topic, time);
- }
- return new PulsarOffsetResponse(offsetResoponse);
- }
-
- /**
- * <pre>
- * Overriden method: OffsetCommitResponse commitOffsets(OffsetCommitRequest request)
- *
- * Note:
- * created PulsarOffsetCommitResponse as OffsetCommitRequest doesn't provide getters
- *
- * </pre>
- */
- public OffsetCommitResponse commitOffsets(PulsarOffsetCommitRequest request) {
-
- PulsarOffsetCommitResponse response = new PulsarOffsetCommitResponse(null);
- for (Entry<String, MessageId> topicOffset : request.getTopicOffsetMap().entrySet()) {
- final String topic = topicOffset.getKey();
- final String groupId = request.getGroupId();
- try {
- Consumer<byte[]> consumer = getConsumer(topic, groupId);
- consumer.acknowledgeCumulative(topicOffset.getValue());
- } catch (Exception e) {
- log.warn("Failed to ack message for topic {}-{}", topic, topicOffset.getValue(), e);
- response.hasError = true;
- TopicAndPartition topicPartition = new TopicAndPartition(topic, 0);
- response.errors.computeIfAbsent(topicPartition, tp -> ErrorMapping.UnknownCode());
- }
- }
-
- return response;
- }
-
- /**
- * <pre>
- * Overriden method: OffsetFetchResponse fetchOffsets(OffsetFetchRequest request)
- *
- * Note:
- * created PulsarOffsetFetchRequest as OffsetFetchRequest doesn't have getters for any field
- * and PulsarOffsetFetchResponse created as base-class doesn't have setters to set state
- * @param request
- * @return
- *
- * </pre>
- */
- public PulsarOffsetFetchResponse fetchOffsets(PulsarOffsetFetchRequest request) {
- final String groupId = request.groupId;
- Map<TopicAndPartition, OffsetMetadataAndError> responseMap = Maps.newHashMap();
- PulsarOffsetFetchResponse response = new PulsarOffsetFetchResponse(responseMap);
- for (TopicAndPartition topicMetadata : request.requestInfo) {
- final String topicName = getTopicName(topicMetadata);
- try {
- PersistentTopicInternalStats stats = admin.topics().getInternalStats(topicName, false);
- CursorStats cursor = stats.cursors != null ? stats.cursors.get(groupId) : null;
- if (cursor != null) {
- String readPosition = cursor.readPosition;
- MessageId msgId = null;
- if (readPosition != null && readPosition.contains(":")) {
- try {
- String[] position = readPosition.split(":");
- msgId = new MessageIdImpl(Long.parseLong(position[0]), Long.parseLong(position[1]), -1);
- } catch (Exception e) {
- log.warn("Invalid read-position {} for {}-{}", readPosition, topicName, groupId);
- }
- }
- msgId = msgId == null ? MessageId.earliest : msgId;
- OffsetMetadataAndError oE = new OffsetMetadataAndError(MessageIdUtils.getOffset(msgId), null,
- ErrorMapping.NoError());
- responseMap.put(topicMetadata, oE);
- }
- } catch (Exception e) {
- OffsetMetadataAndError oE = new OffsetMetadataAndError(0, null, ErrorMapping.UnknownCode());
- responseMap.put(topicMetadata, oE);
- }
- }
- return response;
- }
-
- public static String getTopicName(TopicAndPartition topicMetadata) {
- return topicMetadata.partition() > -1
- ? TopicName.get(topicMetadata.topic()).getPartition(topicMetadata.partition()).toString()
- : topicMetadata.topic();
- }
-
- @Override
- public void close() {
-
- if (topicConsumerMap != null) {
- topicConsumerMap.forEach((topic, consumer) -> {
- try {
- consumer.close();
- } catch (PulsarClientException e) {
- log.warn("Failed to close consumer for topic {}", topic, e);
- }
- });
- topicConsumerMap.clear();
- }
-
- if (client != null) {
- try {
- client.close();
- } catch (PulsarClientException e) {
- log.warn("Failed to close pulsar-client ", e);
- }
- }
-
- if (admin != null) {
- try {
- admin.close();
- } catch (Exception e) {
- log.warn("Failed to close pulsar-admin ", e);
- }
- }
- }
-
- private Consumer<byte[]> getConsumer(String topic, String groupId) {
- TopicGroup topicGroup = new TopicGroup(topic, groupId);
-
- return topicConsumerMap.computeIfAbsent(topicGroup, (topicName) -> {
- try {
- return client.newConsumer().topic(topic).subscriptionName(groupId).subscriptionType(subscriptionType)
- .subscribe();
- } catch (PulsarClientException e) {
- log.error("Failed to create consumer for topic {}", topic, e);
- throw new RuntimeException("Failed to create consumer for topic " + topic, e);
- }
- });
- }
-
- public static class TopicGroup {
- protected String topic;
- protected String grouoId;
-
- public TopicGroup(String topic, String grouoId) {
- super();
- this.topic = topic;
- this.grouoId = grouoId;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(topic, grouoId);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof TopicGroup) {
- TopicGroup t = (TopicGroup) obj;
- return Objects.equals(topic, t.topic) && Objects.equals(grouoId, t.grouoId);
- }
- return false;
- }
-
- }
-
- private SubscriptionType getSubscriptionType(Properties properties) {
- String subType = properties != null && properties.contains(SUBSCRIPTION_TYPE)
- ? properties.getProperty(SUBSCRIPTION_TYPE)
- : SubscriptionType.Failover.toString();
- try {
- return SubscriptionType.valueOf(subType);
- } catch (IllegalArgumentException ie) {
- return SubscriptionType.Failover;
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMessage.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMessage.java
deleted file mode 100644
index b32e39a..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMessage.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.nio.ByteBuffer;
-
-public class PulsarMessage extends kafka.message.Message {
-
- private final String key;
-
- public PulsarMessage(String key, byte[] bytes) {
- super(bytes);
- this.key = key;
- }
-
- @Override
- public ByteBuffer key() {
- return key != null ? ByteBuffer.wrap(key.getBytes()) : null;
- }
-
- @Override
- public boolean hasKey() {
- return key != null;
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMsgAndOffset.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMsgAndOffset.java
deleted file mode 100644
index ebeec88..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarMsgAndOffset.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.util.MessageIdUtils;
-
-import kafka.message.Message;
-import kafka.message.MessageAndOffset;
-
-public class PulsarMsgAndOffset extends MessageAndOffset {
-
- private static final long serialVersionUID = 1L;
- private final MessageId messageId;
-
- public PulsarMsgAndOffset(Message message, MessageId messageId) {
- super(message, MessageIdUtils.getOffset(messageId));
- this.messageId = messageId;
- }
-
- public MessageId getFullOffset() {
- return this.messageId;
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitRequest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitRequest.java
deleted file mode 100644
index ab13c9c..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitRequest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.util.MessageIdUtils;
-
-import com.google.common.collect.Maps;
-
-import kafka.common.OffsetMetadataAndError;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetCommitRequest;
-
-public class PulsarOffsetCommitRequest extends OffsetCommitRequest {
- private final String groupId;
- private final Map<String, MessageId> topicOffsetMap = Maps.newHashMap();
-
- public PulsarOffsetCommitRequest(String groupId, Map<TopicAndPartition, PulsarOffsetMetadataAndError> requestInfo,
- short versionId, int correlationId, String clientId) {
- super(groupId, Collections.emptyMap(), versionId, correlationId, clientId);
- this.groupId = groupId;
- for (Entry<TopicAndPartition, PulsarOffsetMetadataAndError> topicOffset : requestInfo.entrySet()) {
- String topicName = PulsarKafkaSimpleConsumer.getTopicName(topicOffset.getKey());
- OffsetMetadataAndError offsetMetadata = topicOffset.getValue();
- MessageId msgId = null;
- if (offsetMetadata instanceof PulsarOffsetMetadataAndError) {
- msgId = ((PulsarOffsetMetadataAndError) offsetMetadata).getMessageId();
- }
- msgId = msgId == null ? MessageIdUtils.getMessageId(topicOffset.getValue().offset()) : msgId;
- topicOffsetMap.put(topicName, msgId);
- }
- }
-
- public String getGroupId() {
- return groupId;
- }
-
- public Map<String, MessageId> getTopicOffsetMap() {
- return topicOffsetMap;
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitResponse.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitResponse.java
deleted file mode 100644
index e32ffc5..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetCommitResponse.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Map;
-
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetCommitResponse;
-
-public class PulsarOffsetCommitResponse extends OffsetCommitResponse {
-
- protected boolean hasError = false;
- protected Map<TopicAndPartition, Object> errors;
-
- public PulsarOffsetCommitResponse(kafka.api.OffsetCommitResponse underlying) {
- super(underlying);
- }
-
- public boolean hasError() {
- return hasError;
- }
-
- @Override
- public Map<TopicAndPartition, Object> errors() {
- return errors;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchRequest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchRequest.java
deleted file mode 100644
index 98ab6ca..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchRequest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.List;
-
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetFetchRequest;
-
-public class PulsarOffsetFetchRequest extends OffsetFetchRequest {
- protected final String groupId;
- protected final List<TopicAndPartition> requestInfo;
-
- public PulsarOffsetFetchRequest(String groupId, List<TopicAndPartition> requestInfo, short versionId,
- int correlationId, String clientId) {
- super(groupId, requestInfo, versionId, correlationId, clientId);
- this.groupId = groupId;
- this.requestInfo = requestInfo;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchResponse.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchResponse.java
deleted file mode 100644
index cf8d60b..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetFetchResponse.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Map;
-
-import kafka.common.OffsetMetadataAndError;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetFetchResponse;
-
-public class PulsarOffsetFetchResponse extends OffsetFetchResponse {
-
- private final Map<TopicAndPartition, OffsetMetadataAndError> response;
-
- public PulsarOffsetFetchResponse(Map<TopicAndPartition, OffsetMetadataAndError> response) {
- super(null);
- this.response = response;
- }
-
- @Override
- public Map<TopicAndPartition, OffsetMetadataAndError> offsets() {
- return response;
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetMetadataAndError.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetMetadataAndError.java
deleted file mode 100644
index 06e8c3d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetMetadataAndError.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.util.MessageIdUtils;
-
-import kafka.common.OffsetMetadataAndError;
-
-public class PulsarOffsetMetadataAndError extends OffsetMetadataAndError {
-
- private static final long serialVersionUID = 1L;
- private final MessageId messageId;
-
- public PulsarOffsetMetadataAndError(long offset, String metadata, short error) {
- super(offset, metadata, error);
- this.messageId = null;
- }
-
- public PulsarOffsetMetadataAndError(MessageId messageId, String metadata, short error) {
- super(MessageIdUtils.getOffset(messageId), metadata, error);
- this.messageId = messageId;
- }
-
- public MessageId getMessageId() {
- return messageId;
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetRequest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetRequest.java
deleted file mode 100644
index bb7666e..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetRequest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Map;
-
-import kafka.api.PartitionOffsetRequestInfo;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetRequest;
-
-public class PulsarOffsetRequest extends OffsetRequest {
-
- private final Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo;
-
- public PulsarOffsetRequest(Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo, short versionId,
- String clientId) {
- super(requestInfo, versionId, clientId);
- this.requestInfo = requestInfo;
- }
-
- public Map<TopicAndPartition, PartitionOffsetRequestInfo> getRequestInfo() {
- return requestInfo;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetResponse.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetResponse.java
deleted file mode 100644
index f7ce44d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarOffsetResponse.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Map;
-
-import kafka.common.ErrorMapping;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetResponse;
-
-public class PulsarOffsetResponse extends OffsetResponse {
-
- private final Map<TopicAndPartition, Long> offsetResoponse;
-
- public PulsarOffsetResponse(Map<TopicAndPartition, Long> offsetResoponse) {
- super(null);
- this.offsetResoponse = offsetResoponse;
- }
-
- @Override
- public long[] offsets(String topic, int partition) {
- Long offset = offsetResoponse.get(new TopicAndPartition(topic, partition));
- long[] offsets = { offset != null ? offset : 0 };
- return offsets;
- }
-
- @Override
- public boolean hasError() {
- return false;
- }
-
- @Override
- public short errorCode(String topic, int partition) {
- return ErrorMapping.NoError();
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarPartitionMetadata.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarPartitionMetadata.java
deleted file mode 100644
index f61e7d3..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarPartitionMetadata.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Collections;
-import java.util.List;
-
-import kafka.cluster.Broker;
-import kafka.javaapi.PartitionMetadata;
-
-public class PulsarPartitionMetadata extends PartitionMetadata {
-
- private final List<Broker> hosts;
-
- public PulsarPartitionMetadata(String hostUrl, int port) {
- super(null);
- this.hosts = Collections.singletonList(new Broker(0, hostUrl, port));
- }
-
- @Override
- public List<Broker> replicas() {
- return hosts;
- }
-
- @Override
- public Broker leader() {
- return hosts.get(0);
- }
-
- @Override
- public int partitionId() {
- // it always returns partition=-1
- return -1;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadata.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadata.java
deleted file mode 100644
index 4dd6660..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadata.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.Collections;
-import java.util.List;
-
-import kafka.javaapi.PartitionMetadata;
-import kafka.javaapi.TopicMetadata;
-
-public class PulsarTopicMetadata extends TopicMetadata {
-
- private final String hostUrl;
- private final int port;
-
- public PulsarTopicMetadata(String hostUrl, int port, String topic) {
- super(null);
- this.hostUrl = hostUrl;
- this.port = port;
- }
-
- @Override
- public List<PartitionMetadata> partitionsMetadata() {
- return Collections.singletonList(new PulsarPartitionMetadata(hostUrl, port));
- }
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadataResponse.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadataResponse.java
deleted file mode 100644
index e1d39c5..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/main/java/org/apache/kafka/clients/simple/consumer/PulsarTopicMetadataResponse.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.simple.consumer;
-
-import java.util.List;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.common.naming.TopicName;
-
-import com.google.common.collect.Lists;
-
-import kafka.javaapi.TopicMetadata;
-import kafka.javaapi.TopicMetadataResponse;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class PulsarTopicMetadataResponse extends TopicMetadataResponse {
-
- private final List<String> topics;
- private final String hostUrl;
- private final int port;
- private final PulsarAdmin admin;
-
- public PulsarTopicMetadataResponse(PulsarAdmin admin, String hostUrl, int port, List<String> topics) {
- super(null);
- this.hostUrl = hostUrl;
- this.port = port;
- this.topics = topics;
- this.admin = admin;
- }
-
- @Override
- public List<TopicMetadata> topicsMetadata() {
- List<TopicMetadata> metadataList = Lists.newArrayList();
- topics.forEach(topic -> {
- try {
- int partitions;
- partitions = admin.topics().getPartitionedTopicMetadata(topic).partitions;
- if (partitions > 0) {
- for (int partition = 0; partition < partitions; partition++) {
- String topicName = TopicName.get(topic).getPartition(partition).toString();
- metadataList.add(new PulsarTopicMetadata(hostUrl, port, topicName));
- }
- } else {
- metadataList.add(new PulsarTopicMetadata(hostUrl, port, topic));
- }
- } catch (PulsarAdminException e) {
- log.error("Failed to get partitioned metadata for {}", topic, e);
- throw new RuntimeException("Failed to get partitioned-metadata", e);
- }
- });
- return metadataList;
- }
-
-}
\ No newline at end of file
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumerTest.java
deleted file mode 100644
index 77d7e8c..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumerTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import static org.testng.Assert.assertEquals;
-
-import java.lang.reflect.Field;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.impl.ConsumerBuilderImpl;
-import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
-import org.testng.annotations.Test;
-
-import kafka.consumer.ConsumerConfig;
-
-public class PulsarKafkaConsumerTest {
-
- @Test
- public void testPulsarKafkaConsumerWithDefaultConfig() throws Exception {
- // https://kafka.apache.org/08/documentation.html#consumerconfigs
- Properties properties = new Properties();
- properties.put("zookeeper.connect", "http://localhost:8080/");
- properties.put("group.id", "group1");
-
- ConsumerConfig config = new ConsumerConfig(properties);
- ConsumerConnector connector = new ConsumerConnector(config);
- ConsumerBuilderImpl<byte[]> consumerBuilder = (ConsumerBuilderImpl<byte[]>) connector.getConsumerBuilder();
- Field confField = consumerBuilder.getClass().getDeclaredField("conf");
- confField.setAccessible(true);
- ConsumerConfigurationData conf = (ConsumerConfigurationData) confField.get(consumerBuilder);
- assertEquals(conf.getSubscriptionName(), "group1");
- assertEquals(conf.getReceiverQueueSize(), 1000);
- }
-
- @Test
- public void testPulsarKafkaConsumerConfig() throws Exception {
- // https://kafka.apache.org/08/documentation.html#consumerconfigs
- Properties properties = new Properties();
- properties.put("zookeeper.connect", "http://localhost:8080/");
- properties.put("group.id", "group1");
- properties.put("consumer.id", "cons1");
- properties.put("auto.commit.enable", "true");
- properties.put("auto.commit.interval.ms", "100");
- properties.put("queued.max.message.chunks", "100");
-
- ConsumerConfig config = new ConsumerConfig(properties);
- ConsumerConnector connector = new ConsumerConnector(config);
- ConsumerBuilderImpl<byte[]> consumerBuilder = (ConsumerBuilderImpl<byte[]>) connector.getConsumerBuilder();
- Field confField = consumerBuilder.getClass().getDeclaredField("conf");
- confField.setAccessible(true);
- ConsumerConfigurationData conf = (ConsumerConfigurationData) confField.get(consumerBuilder);
- assertEquals(conf.getSubscriptionName(), "group1");
- assertEquals(conf.getReceiverQueueSize(), 100);
- assertEquals(conf.getAcknowledgementsGroupTimeMicros(), TimeUnit.MILLISECONDS.toMicros(100));
- System.out.println(conf);
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
deleted file mode 100644
index 4cccf83..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import static org.testng.Assert.assertEquals;
-
-import java.lang.reflect.Field;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.CompressionType;
-import org.apache.pulsar.client.impl.ProducerBuilderImpl;
-import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
-import org.testng.annotations.Test;
-
-import kafka.producer.Partitioner;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.Encoder;
-
-public class PulsarKafkaProducerTest {
-
- private static final String BROKER_URL = "metadata.broker.list";
- private static final String PRODUCER_TYPE = "producer.type";
- private static final String SERIALIZER_CLASS = "serializer.class";
- private static final String KEY_SERIALIZER_CLASS = "key.serializer.class";
- private static final String PARTITIONER_CLASS = "partitioner.class";
- private static final String COMPRESSION_CODEC = "compression.codec";
- private static final String QUEUE_BUFFERING_MAX_MS = "queue.buffering.max.ms";
- private static final String QUEUE_BUFFERING_MAX_MESSAGES = "queue.buffering.max.messages";
- private static final String QUEUE_ENQUEUE_TIMEOUT_MS = "queue.enqueue.timeout.ms";
- private static final String BATCH_NUM_MESSAGES = "batch.num.messages";
- private static final String CLIENT_ID = "client.id";
-
- @Test
- public void testPulsarKafkaProducerWithDefaultConfig() throws Exception {
- // https://kafka.apache.org/08/documentation.html#producerconfigs
- Properties properties = new Properties();
- properties.put(BROKER_URL, "http://localhost:8080/");
-
- ProducerConfig config = new ProducerConfig(properties);
- PulsarKafkaProducer<byte[], byte[]> producer = new PulsarKafkaProducer<>(config);
- ProducerBuilderImpl<byte[]> producerBuilder = (ProducerBuilderImpl<byte[]>) producer.getPulsarProducerBuilder();
- Field field = ProducerBuilderImpl.class.getDeclaredField("conf");
- field.setAccessible(true);
- ProducerConfigurationData conf = (ProducerConfigurationData) field.get(producerBuilder);
- System.out.println("getMaxPendingMessages= " + conf.getMaxPendingMessages());
- assertEquals(conf.getCompressionType(), CompressionType.NONE);
- assertEquals(conf.isBlockIfQueueFull(), true);
- assertEquals(conf.getMaxPendingMessages(), 1000);
- assertEquals(conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MILLISECONDS.toMicros(1));
- assertEquals(conf.getBatchingMaxMessages(), 1000);
- }
-
- @Test
- public void testPulsarKafkaProducer() throws Exception {
- // https://kafka.apache.org/08/documentation.html#producerconfigs
- Properties properties = new Properties();
- properties.put(BROKER_URL, "http://localhost:8080/");
- properties.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
- properties.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
- properties.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
- properties.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
- properties.put(BATCH_NUM_MESSAGES, "500"); // batch msg
- properties.put(CLIENT_ID, "test");
- ProducerConfig config = new ProducerConfig(properties);
- PulsarKafkaProducer<byte[], byte[]> producer = new PulsarKafkaProducer<>(config);
- ProducerBuilderImpl<byte[]> producerBuilder = (ProducerBuilderImpl<byte[]>) producer.getPulsarProducerBuilder();
- Field field = ProducerBuilderImpl.class.getDeclaredField("conf");
- field.setAccessible(true);
- ProducerConfigurationData conf = (ProducerConfigurationData) field.get(producerBuilder);
- assertEquals(conf.getCompressionType(), CompressionType.ZLIB);
- assertEquals(conf.isBlockIfQueueFull(), true);
- assertEquals(conf.getMaxPendingMessages(), 6000);
- assertEquals(conf.getBatchingMaxPublishDelayMicros(), TimeUnit.MILLISECONDS.toMicros(100));
- assertEquals(conf.getBatchingMaxMessages(), 500);
- }
-
- @Test
- public void testPulsarKafkaProducerWithSerializer() throws Exception {
- Properties properties = new Properties();
- properties.put(BROKER_URL, "http://localhost:8080/");
- properties.put(PRODUCER_TYPE, "sync");
- properties.put(SERIALIZER_CLASS, TestEncoder.class.getName());
- properties.put(KEY_SERIALIZER_CLASS, TestEncoder.class.getName());
- properties.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
- ProducerConfig config = new ProducerConfig(properties);
- PulsarKafkaProducer<byte[], byte[]> producer = new PulsarKafkaProducer<>(config);
- assertEquals(producer.getKeySerializer().getClass(), TestEncoder.class);
- assertEquals(producer.getValueSerializer().getClass(), TestEncoder.class);
- assertEquals(producer.getPartitioner().getClass(), TestPartitioner.class);
-
- }
-
- public static class TestEncoder implements Encoder<String> {
- @Override
- public byte[] toBytes(String value) {
- return value.getBytes();
- }
- }
-
- public static class TestPartitioner implements Partitioner {
- @Override
- public int partition(Object obj, int totalPartition) {
- return obj.hashCode() % totalPartition;
- }
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerConsumerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerConsumerTest.java
deleted file mode 100644
index bbf61be..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerConsumerTest.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.test;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.kafka.clients.consumer.ConsumerConnector;
-import org.apache.kafka.clients.consumer.PulsarKafkaStream;
-import org.apache.kafka.clients.consumer.PulsarMessageAndMetadata;
-import org.apache.kafka.clients.producer.PulsarKafkaProducer;
-import org.apache.pulsar.client.api.ProducerConsumerBase;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Sets;
-
-import kafka.consumer.ConsumerConfig;
-import kafka.producer.KeyedMessage;
-import kafka.producer.Partitioner;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.Decoder;
-import kafka.serializer.Encoder;
-import kafka.serializer.StringDecoder;
-import kafka.serializer.StringEncoder;
-
-public class KafkaProducerConsumerTest extends ProducerConsumerBase {
-
- private static final String BROKER_URL = "metadata.broker.list";
- private static final String PRODUCER_TYPE = "producer.type";
- private static final String SERIALIZER_CLASS = "serializer.class";
- private static final String KEY_SERIALIZER_CLASS = "key.serializer.class";
- private static final String PARTITIONER_CLASS = "partitioner.class";
- private static final String COMPRESSION_CODEC = "compression.codec";
- private static final String QUEUE_BUFFERING_MAX_MS = "queue.buffering.max.ms";
- private static final String QUEUE_BUFFERING_MAX_MESSAGES = "queue.buffering.max.messages";
- private static final String QUEUE_ENQUEUE_TIMEOUT_MS = "queue.enqueue.timeout.ms";
- private static final String BATCH_NUM_MESSAGES = "batch.num.messages";
- private static final String CLIENT_ID = "client.id";
-
- @BeforeMethod
- @Override
- protected void setup() throws Exception {
- super.internalSetup();
- super.producerBaseSetup();
- }
-
- @AfterMethod
- @Override
- protected void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- @Test
- public void testPulsarKafkaProducerWithSerializer() throws Exception {
- final String serviceUrl = lookupUrl.toString();
- final String topicName = "persistent://my-property/my-ns/my-topic1";
-
- // (1) Create consumer
- Properties properties = new Properties();
- properties.put("zookeeper.connect", serviceUrl);
- properties.put("group.id", "group1");
- properties.put("consumer.id", "cons1");
- properties.put("auto.commit.enable", "true");
- properties.put("auto.commit.interval.ms", "100");
- properties.put("queued.max.message.chunks", "100");
-
- ConsumerConfig conSConfig = new ConsumerConfig(properties);
- ConsumerConnector connector = new ConsumerConnector(conSConfig);
- Map<String, Integer> topicCountMap = Collections.singletonMap(topicName, 2);
- Map<String, List<PulsarKafkaStream<String, Tweet>>> streams = connector.createMessageStreams(topicCountMap,
- new StringDecoder(null), new TestDecoder());
-
- // (2) Create producer
- Properties properties2 = new Properties();
- properties2.put(BROKER_URL, serviceUrl);
- properties2.put(PRODUCER_TYPE, "sync");
- properties2.put(SERIALIZER_CLASS, TestEncoder.class.getName());
- properties2.put(KEY_SERIALIZER_CLASS, StringEncoder.class.getName());
- properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
- properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
- properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
- properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
- properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
- properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
- properties2.put(CLIENT_ID, "test");
- ProducerConfig config = new ProducerConfig(properties2);
- PulsarKafkaProducer<String, Tweet> producer = new PulsarKafkaProducer<>(config);
-
- String name = "user";
- String msg = "Hello World!";
- Set<Tweet> published = Sets.newHashSet();
- Set<Tweet> received = Sets.newHashSet();
- int total = 10;
- for (int i = 0; i < total; i++) {
- String sendMessage = msg + i;
- Tweet tweet = new Tweet(name, sendMessage);
- KeyedMessage<String, Tweet> message = new KeyedMessage<>(topicName, name, tweet);
- published.add(tweet);
- producer.send(message);
- }
- while (received.size() < total) {
- for (int i = 0; i < streams.size(); i++) {
- List<PulsarKafkaStream<String, Tweet>> kafkaStreams = streams.get(topicName);
- assertEquals(kafkaStreams.size(), 2);
- for (PulsarKafkaStream<String, Tweet> kafkaStream : kafkaStreams) {
- for (PulsarMessageAndMetadata<String, KafkaProducerConsumerTest.Tweet> record : kafkaStream) {
- received.add(record.message());
- assertEquals(record.key(), name);
- }
- }
- }
- }
- assertEquals(published.size(), received.size());
- published.removeAll(received);
- assertTrue(published.isEmpty());
- }
-
- public static class Tweet implements Serializable {
- private static final long serialVersionUID = 1L;
- public String userName;
- public String message;
-
- public Tweet(String userName, String message) {
- super();
- this.userName = userName;
- this.message = message;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(userName, message);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof Tweet) {
- Tweet tweet = (Tweet) obj;
- return Objects.equal(this.userName, tweet.userName) && Objects.equal(this.message, tweet.message);
- }
- return false;
- }
- }
-
- public static class TestEncoder implements Encoder<Tweet> {
- @Override
- public byte[] toBytes(Tweet tweet) {
- return (tweet.userName + "," + tweet.message).getBytes();
- }
- }
-
- public static class TestDecoder implements Decoder<Tweet> {
- @Override
- public Tweet fromBytes(byte[] input) {
- String[] tokens = (new String(input)).split(",");
- return new Tweet(tokens[0], tokens[1]);
- }
- }
-
- public static class TestPartitioner implements Partitioner {
- @Override
- public int partition(Object obj, int totalPartition) {
- return obj.hashCode() % totalPartition;
- }
- }
-
-
- @Test
- public void testProducerConsumerWithoutSerializer() throws Exception {
- final String serviceUrl = lookupUrl.toString();
- final String topicName = "persistent://my-property/my-ns/my-topic1";
- // (1) Create consumer
- Properties properties = new Properties();
- properties.put("zookeeper.connect", serviceUrl);
- properties.put("group.id", "group1");
- properties.put("consumer.id", "cons1");
- properties.put("auto.commit.enable", "true");
- properties.put("auto.commit.interval.ms", "100");
- properties.put("queued.max.message.chunks", "100");
-
- ConsumerConfig conSConfig = new ConsumerConfig(properties);
- ConsumerConnector connector = new ConsumerConnector(conSConfig);
- Map<String, Integer> topicCountMap = Collections.singletonMap(topicName, 2);
- Map<String, List<PulsarKafkaStream<byte[], byte[]>>> streams = connector.createMessageStreams(topicCountMap);
-
- // (2) Create producer
- Properties properties2 = new Properties();
- properties2.put(BROKER_URL, serviceUrl);
- properties2.put(PRODUCER_TYPE, "sync");
- properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
- properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
- properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
- properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
- properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
- properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
- properties2.put(CLIENT_ID, "test");
- ProducerConfig config = new ProducerConfig(properties2);
- PulsarKafkaProducer<byte[], byte[]> producer = new PulsarKafkaProducer<>(config);
-
- String name = "user";
- String msg = "Hello World!";
- int total = 10;
- for (int i = 0; i < total; i++) {
- String sendMessage = msg + i;
- KeyedMessage<byte[], byte[]> message = new KeyedMessage<>(topicName, name.getBytes(), sendMessage.getBytes());
- producer.send(message);
- }
- int count = 0;
- while (count < total) {
- for (int i = 0; i < streams.size(); i++) {
- List<PulsarKafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
- assertEquals(kafkaStreams.size(), 2);
- for (PulsarKafkaStream<byte[], byte[]> kafkaStream : kafkaStreams) {
- for (PulsarMessageAndMetadata<byte[], byte[]> record : kafkaStream) {
- count++;
- System.out.println(new String(record.message()));
- }
- }
- }
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerSimpleConsumerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerSimpleConsumerTest.java
deleted file mode 100644
index 82621fb..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_8/src/test/java/org/apache/pulsar/client/kafka/test/KafkaProducerSimpleConsumerTest.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.test;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.kafka.clients.producer.PulsarKafkaProducer;
-import org.apache.kafka.clients.simple.consumer.PulsarKafkaSimpleConsumer;
-import org.apache.kafka.clients.simple.consumer.PulsarMsgAndOffset;
-import org.apache.kafka.clients.simple.consumer.PulsarOffsetCommitRequest;
-import org.apache.kafka.clients.simple.consumer.PulsarOffsetFetchRequest;
-import org.apache.kafka.clients.simple.consumer.PulsarOffsetMetadataAndError;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.ProducerConsumerBase;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Sets;
-
-import kafka.api.FetchRequest;
-import kafka.api.FetchRequestBuilder;
-import kafka.common.OffsetMetadataAndError;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.PartitionMetadata;
-import kafka.javaapi.TopicMetadata;
-import kafka.javaapi.TopicMetadataRequest;
-import kafka.message.MessageAndOffset;
-import kafka.producer.KeyedMessage;
-import kafka.producer.Partitioner;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.Decoder;
-import kafka.serializer.Encoder;
-import kafka.serializer.StringEncoder;
-
-public class KafkaProducerSimpleConsumerTest extends ProducerConsumerBase {
-
- private static final String BROKER_URL = "metadata.broker.list";
- private static final String PRODUCER_TYPE = "producer.type";
- private static final String KEY_SERIALIZER_CLASS = "key.serializer.class";
- private static final String PARTITIONER_CLASS = "partitioner.class";
- private static final String COMPRESSION_CODEC = "compression.codec";
- private static final String QUEUE_BUFFERING_MAX_MS = "queue.buffering.max.ms";
- private static final String QUEUE_BUFFERING_MAX_MESSAGES = "queue.buffering.max.messages";
- private static final String QUEUE_ENQUEUE_TIMEOUT_MS = "queue.enqueue.timeout.ms";
- private static final String BATCH_NUM_MESSAGES = "batch.num.messages";
- private static final String CLIENT_ID = "client.id";
-
- private final static int publishPartition = 1;
-
- @DataProvider(name = "partitions")
- public Object[][] totalPartitions() {
- return new Object[][] { { 0 }, { 10 } };
- }
-
- @BeforeMethod
- @Override
- protected void setup() throws Exception {
- super.internalSetup();
- super.producerBaseSetup();
- }
-
- @AfterMethod
- @Override
- protected void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- @Test(dataProvider="partitions")
- public void testPulsarKafkaProducerWithSerializer(int partitions) throws Exception {
- final String serviceUrl = lookupUrl.toString();
- final String topicName = "persistent://my-property/my-ns/my-topic";
- final String groupId = "group1";
-
- int partition = -1;
- if (partitions > 0) {
- admin.topics().createPartitionedTopic(topicName, 10);
- partition = publishPartition;
- }
-
- // create subscription
- Consumer<byte[]> cons = pulsarClient.newConsumer().topic(topicName).subscriptionName(groupId).subscribe();
- cons.close();
-
- // (2) Create producer
- Properties properties2 = new Properties();
- properties2.put(BROKER_URL, serviceUrl);
- properties2.put(PRODUCER_TYPE, "sync");
- properties2.put(KEY_SERIALIZER_CLASS, StringEncoder.class.getName());
- properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
- properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
- properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
- properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
- properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
- properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
- properties2.put(CLIENT_ID, "test");
- ProducerConfig config = new ProducerConfig(properties2);
- PulsarKafkaProducer<String, byte[]> producer = new PulsarKafkaProducer<>(config);
-
- String name = "user";
- String msg = "Hello World!";
- Set<String> published = Sets.newHashSet();
- Set<String> received = Sets.newHashSet();
- int total = 10;
- for (int i = 0; i < total; i++) {
- String sendMessage = msg + i;
- KeyedMessage<String, byte[]> message = new KeyedMessage<>(topicName, name, sendMessage.getBytes());
- published.add(sendMessage);
- producer.send(message);
- }
-
-
- // (2) Consume using simple consumer
- PulsarKafkaSimpleConsumer consumer = new PulsarKafkaSimpleConsumer(serviceUrl, 0, 0, 0, "clientId");
- List<String> topics = Collections.singletonList(topicName);
- TopicMetadataRequest req = new TopicMetadataRequest(topics);
- kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
-
- List<TopicMetadata> metaData = resp.topicsMetadata();
- PartitionMetadata part = metaData.get(0).partitionsMetadata().get(0);
-
- long readOffset = kafka.api.OffsetRequest.EarliestTime();
- FetchRequest fReq = new FetchRequestBuilder()
- .clientId("c1")
- .addFetch(topicName, partition, readOffset, 100000)
- .build();
- FetchResponse fetchResponse = consumer.fetch(fReq);
-
- long lastOffset = 0;
- MessageId offset = null;
- for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topicName, partition)) {
- long currentOffset = messageAndOffset.offset();
- if (currentOffset < readOffset) {
- continue;
- }
- offset = ((PulsarMsgAndOffset)messageAndOffset).getFullOffset();
- lastOffset = messageAndOffset.offset();
- ByteBuffer payload = messageAndOffset.message().payload();
-
- byte[] bytes = new byte[payload.limit()];
- payload.get(bytes);
- received.add(new String(bytes, StandardCharsets.UTF_8));
- }
- lastOffset -= 1;
-
- assertEquals(published.size(), received.size());
- published.removeAll(received);
- assertTrue(published.isEmpty());
-
- TopicAndPartition topicPartition = new TopicAndPartition(topicName, partition);
- PulsarOffsetMetadataAndError offsetError = new PulsarOffsetMetadataAndError(offset, null, (short) 0);
- Map<TopicAndPartition, PulsarOffsetMetadataAndError> requestInfo = Collections.singletonMap(topicPartition,
- offsetError);
- PulsarOffsetCommitRequest offsetReq = new PulsarOffsetCommitRequest(groupId, requestInfo, (short) -1, 0, "c1");
- consumer.commitOffsets(offsetReq);
-
- final long expectedReadOffsetPosition = lastOffset;
-
- retryStrategically((test) -> fetchOffset(consumer, topicPartition, groupId) == expectedReadOffsetPosition, 10, 150);
-
- long offset1 = fetchOffset(consumer, topicPartition, groupId);
- MessageIdImpl actualMsgId = ((MessageIdImpl)MessageIdUtils.getMessageId(offset1));
-
- MessageIdImpl expectedMsgId = (MessageIdImpl) offset;
- assertEquals(actualMsgId.getLedgerId(), expectedMsgId.getLedgerId());
- assertEquals(actualMsgId.getEntryId(), expectedMsgId.getEntryId() + 1);
- }
-
- private long fetchOffset(PulsarKafkaSimpleConsumer consumer, TopicAndPartition topicPartition, String groupId) {
- List<TopicAndPartition> fetchReqInfo = Collections.singletonList(topicPartition);
- PulsarOffsetFetchRequest fetchOffsetRequest = new PulsarOffsetFetchRequest(groupId, fetchReqInfo, (short)-1, 0, "test");
- OffsetMetadataAndError offsetResponse = consumer.fetchOffsets(fetchOffsetRequest).offsets().get(topicPartition);
- return offsetResponse.offset();
- }
-
- public static class Tweet implements Serializable {
- private static final long serialVersionUID = 1L;
- public String userName;
- public String message;
-
- public Tweet(String userName, String message) {
- super();
- this.userName = userName;
- this.message = message;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(userName, message);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof Tweet) {
- Tweet tweet = (Tweet) obj;
- return Objects.equal(this.userName, tweet.userName) && Objects.equal(this.message, tweet.message);
- }
- return false;
- }
- }
-
- public static class TestEncoder implements Encoder<Tweet> {
- @Override
- public byte[] toBytes(Tweet tweet) {
- return (tweet.userName + "," + tweet.message).getBytes();
- }
- }
-
- public static class TestDecoder implements Decoder<Tweet> {
- @Override
- public Tweet fromBytes(byte[] input) {
- String[] tokens = (new String(input)).split(",");
- return new Tweet(tokens[0], tokens[1]);
- }
- }
-
- public static class TestPartitioner implements Partitioner {
- @Override
- public int partition(Object obj, int totalPartition) {
- //return obj.hashCode() % totalPartition;
- return publishPartition;
- }
- }
-
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/pom.xml
deleted file mode 100644
index a4a9506..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/pom.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-kafka-compat</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <kafka_0_9.version>0.9.0.1</kafka_0_9.version>
- </properties>
-
- <artifactId>pulsar-client-kafka_0_9-original</artifactId>
- <name>Pulsar Kafka compatibility 0.9 :: API (original)</name>
-
- <description>Kafka client library that publishes and consumes messages on Pulsar topics</description>
-
- <dependencies>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka_0_9.version}</version>
- <exclusions>
- <exclusion>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
deleted file mode 100644
index 0a3526b..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/consumer/PulsarKafkaConsumer.java
+++ /dev/null
@@ -1,596 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.consumer;
-
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.annotation.InterfaceStability;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.pulsar.client.api.MessageListener;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.SubscriptionInitialPosition;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarConsumerKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarKafkaSchema;
-import org.apache.pulsar.client.util.ConsumerName;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
-
-@Slf4j
-public class PulsarKafkaConsumer<K, V> implements Consumer<K, V>, MessageListener<byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- private final PulsarClient client;
-
- private final Schema<K> keySchema;
- private final Schema<V> valueSchema;
-
- private final String groupId;
- private final boolean isAutoCommit;
-
- private final ConcurrentMap<TopicPartition, org.apache.pulsar.client.api.Consumer<byte[]>> consumers = new ConcurrentHashMap<>();
-
- private final Map<TopicPartition, Long> lastReceivedOffset = new ConcurrentHashMap<>();
- private final Map<TopicPartition, OffsetAndMetadata> lastCommittedOffset = new ConcurrentHashMap<>();
- private final Set<TopicPartition> unpolledPartitions = new HashSet<>();
- private final SubscriptionInitialPosition strategy;
-
- private volatile boolean closed = false;
-
- private final int maxRecordsInSinglePoll;
-
- private final Properties properties;
-
-
- private static class QueueItem {
- final org.apache.pulsar.client.api.Consumer<byte[]> consumer;
- final Message<byte[]> message;
-
- QueueItem(org.apache.pulsar.client.api.Consumer<byte[]> consumer, Message<byte[]> message) {
- this.consumer = consumer;
- this.message = message;
- }
- }
-
- // Since a single Kafka consumer can receive from multiple topics, we need to multiplex all the different
- // topics/partitions into a single queues
- private final BlockingQueue<QueueItem> receivedMessages = new ArrayBlockingQueue<>(1000);
-
- public PulsarKafkaConsumer(Map<String, Object> configs) {
- this(new ConsumerConfig(configs), null, null);
- }
-
- public PulsarKafkaConsumer(Map<String, Object> configs, Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer) {
- this(new ConsumerConfig(configs),
- new PulsarKafkaSchema<>(keyDeserializer), new PulsarKafkaSchema<>(valueDeserializer));
- }
-
- public PulsarKafkaConsumer(Map<String, Object> configs, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ConsumerConfig(configs), keySchema, valueSchema);
- }
-
- public PulsarKafkaConsumer(Properties properties) {
- this(new ConsumerConfig(properties), null, null);
- }
-
- public PulsarKafkaConsumer(Properties properties, Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer) {
- this(new ConsumerConfig(properties),
- new PulsarKafkaSchema<>(keyDeserializer), new PulsarKafkaSchema<>(valueDeserializer));
- }
-
- public PulsarKafkaConsumer(Properties properties, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ConsumerConfig(properties), keySchema, valueSchema);
- }
-
- @SuppressWarnings("unchecked")
- private PulsarKafkaConsumer(ConsumerConfig consumerConfig, Schema<K> keySchema, Schema<V> valueSchema) {
-
- if (keySchema == null) {
- Deserializer<K> kafkaKeyDeserializer = consumerConfig.getConfiguredInstance(
- ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, Deserializer.class);
- kafkaKeyDeserializer.configure(consumerConfig.originals(), true);
- this.keySchema = new PulsarKafkaSchema<>(kafkaKeyDeserializer);
- } else {
- this.keySchema = keySchema;
- consumerConfig.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG);
- }
-
- if (valueSchema == null) {
- Deserializer<V> kafkaValueDeserializer = consumerConfig.getConfiguredInstance(
- ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, Deserializer.class);
- kafkaValueDeserializer.configure(consumerConfig.originals(), true);
- this.valueSchema = new PulsarKafkaSchema<>(kafkaValueDeserializer);
- } else {
- this.valueSchema = valueSchema;
- consumerConfig.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG);
- }
-
- groupId = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG);
- isAutoCommit = consumerConfig.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
- strategy = getStrategy(consumerConfig.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
- log.info("Offset reset strategy has been assigned value {}", strategy);
-
- String serviceUrl = consumerConfig.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG).get(0);
-
- // there is not this config in kafka 0.9, so use default value.
- maxRecordsInSinglePoll = 1000;
-
- this.properties = new Properties();
- consumerConfig.originals().forEach(properties::put);
- ClientBuilder clientBuilder = PulsarClientKafkaConfig.getClientBuilder(properties);
- // Since this client instance is going to be used just for the consumers, we can enable Nagle to group
- // all the acknowledgments sent to broker within a short time frame
- clientBuilder.enableTcpNoDelay(false);
- try {
- client = clientBuilder.serviceUrl(serviceUrl).build();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- private SubscriptionInitialPosition getStrategy(final String strategy) {
- if ("earliest".equals(strategy)) {
- return SubscriptionInitialPosition.Earliest;
- } else {
- return SubscriptionInitialPosition.Latest;
- }
- }
-
- @Override
- public Set<TopicPartition> assignment() {
- throw new UnsupportedOperationException("Cannot access the partitions assignements");
- }
-
- @Override
- public Set<String> subscription() {
- return consumers.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
- }
-
- @Override
- public void subscribe(List<String> topics) {
- subscribe(topics, null);
- }
-
- @Override
- public void subscribe(List<String> topics, ConsumerRebalanceListener callback) {
- List<CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>>> futures = new ArrayList<>();
-
- List<TopicPartition> topicPartitions = new ArrayList<>();
- try {
- for (String topic : topics) {
- // Create individual subscription on each partition, that way we can keep using the
- // acknowledgeCumulative()
- int numberOfPartitions = ((PulsarClientImpl) client).getNumberOfPartitions(topic).get();
-
- ConsumerBuilder<byte[]> consumerBuilder = PulsarConsumerKafkaConfig.getConsumerBuilder(client, properties);
- consumerBuilder.subscriptionType(SubscriptionType.Failover);
- consumerBuilder.messageListener(this);
- consumerBuilder.subscriptionName(groupId);
- if (numberOfPartitions > 1) {
- // Subscribe to each partition
- consumerBuilder.consumerName(ConsumerName.generateRandomName());
- for (int i = 0; i < numberOfPartitions; i++) {
- String partitionName = TopicName.get(topic).getPartition(i).toString();
- CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.clone()
- .topic(partitionName).subscribeAsync();
- int partitionIndex = i;
- TopicPartition tp = new TopicPartition(
- TopicName.get(topic).getPartitionedTopicName(),
- partitionIndex);
- futures.add(future.thenApply(consumer -> {
- log.info("Add consumer {} for partition {}", consumer, tp);
- consumers.putIfAbsent(tp, consumer);
- return consumer;
- }));
- topicPartitions.add(tp);
- }
- } else {
- // Topic has a single partition
- CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.topic(topic)
- .subscribeAsync();
- TopicPartition tp = new TopicPartition(
- TopicName.get(topic).getPartitionedTopicName(),
- 0);
- futures.add(future.thenApply(consumer -> {
- log.info("Add consumer {} for partition {}", consumer, tp);
- consumers.putIfAbsent(tp, consumer);
- return consumer;
- }));
- topicPartitions.add(tp);
- }
- }
- unpolledPartitions.addAll(topicPartitions);
-
- // Wait for all consumers to be ready
- futures.forEach(CompletableFuture::join);
-
- // Notify the listener is now owning all topics/partitions
- if (callback != null) {
- callback.onPartitionsAssigned(topicPartitions);
- }
-
- } catch (Exception e) {
- // Close all consumer that might have been successfully created
- futures.forEach(f -> {
- try {
- f.get().close();
- } catch (Exception e1) {
- // Ignore. Consumer already had failed
- }
- });
-
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void assign(List<TopicPartition> list) {
- throw new UnsupportedOperationException("Cannot manually assign partitions");
- }
-
- @Override
- public void subscribe(Pattern pattern, ConsumerRebalanceListener consumerRebalanceListener) {
- throw new UnsupportedOperationException("Cannot subscribe with topic name pattern");
- }
-
- @Override
- public void unsubscribe() {
- consumers.values().forEach(c -> {
- try {
- c.unsubscribe();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- });
- }
-
- @Override
- public ConsumerRecords<K, V> poll(long timeoutMillis) {
- try {
- QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
- if (item == null) {
- return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
- }
-
- Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();
-
- int numberOfRecords = 0;
-
- while (item != null) {
- TopicName topicName = TopicName.get(item.consumer.getTopic());
- String topic = topicName.getPartitionedTopicName();
- int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
- Message<byte[]> msg = item.message;
- MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
- long offset = MessageIdUtils.getOffset(msgId);
-
- TopicPartition tp = new TopicPartition(topic, partition);
- if (lastReceivedOffset.get(tp) == null && !unpolledPartitions.contains(tp)) {
- log.info("When polling offsets, invalid offsets were detected. Resetting topic partition {}", tp);
- resetOffsets(tp);
- }
-
- K key = getKey(topic, msg);
- if (valueSchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<V>) valueSchema).setTopic(topic);
- }
- V value = valueSchema.decode(msg.getData());
-
- ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, key, value);
-
- records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);
-
- // Update last offset seen by application
- lastReceivedOffset.put(tp, offset);
- unpolledPartitions.remove(tp);
-
- if (++numberOfRecords >= maxRecordsInSinglePoll) {
- break;
- }
-
- // Check if we have an item already available
- item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
- }
-
- if (isAutoCommit && !records.isEmpty()) {
- // Commit the offset of previously dequeued messages
- commitAsync();
- }
-
- return new ConsumerRecords<>(records);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void commitSync() {
- try {
- doCommitOffsets(getCurrentOffsetsMap()).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) {
- try {
- doCommitOffsets(offsets).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void commitAsync() {
- doCommitOffsets(getCurrentOffsetsMap());
- }
-
- @Override
- public void commitAsync(OffsetCommitCallback callback) {
- Map<TopicPartition, OffsetAndMetadata> offsets = getCurrentOffsetsMap();
- doCommitOffsets(offsets).handle((v, throwable) -> {
- callback.onComplete(offsets, throwable != null ? new Exception(throwable) : null);
- return null;
- });
- }
-
- @Override
- public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
- doCommitOffsets(offsets).handle((v, throwable) -> {
- callback.onComplete(offsets, throwable != null ? new Exception(throwable) : null);
- return null;
- });
- }
-
- private CompletableFuture<Void> doCommitOffsets(Map<TopicPartition, OffsetAndMetadata> offsets) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- offsets.forEach((topicPartition, offsetAndMetadata) -> {
- org.apache.pulsar.client.api.Consumer<byte[]> consumer = consumers.get(topicPartition);
- lastCommittedOffset.put(topicPartition, offsetAndMetadata);
- futures.add(consumer.acknowledgeCumulativeAsync(MessageIdUtils.getMessageId(offsetAndMetadata.offset())));
- });
-
- return FutureUtil.waitForAll(futures);
- }
-
- private Map<TopicPartition, OffsetAndMetadata> getCurrentOffsetsMap() {
- Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
- lastReceivedOffset.forEach((topicPartition, offset) -> {
- OffsetAndMetadata om = new OffsetAndMetadata(offset);
- offsets.put(topicPartition, om);
- });
-
- return offsets;
- }
-
- @Override
- public void seek(TopicPartition partition, long offset) {
- MessageId msgId = MessageIdUtils.getMessageId(offset);
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(partition);
- if (c == null) {
- throw new IllegalArgumentException("Cannot seek on a partition where we are not subscribed");
- }
-
- try {
- c.seek(msgId);
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void seekToBeginning(TopicPartition... partitions) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- if (partitions.length == 0) {
- partitions = consumers.keySet().toArray(new TopicPartition[0]);
- }
- lastCommittedOffset.clear();
- lastReceivedOffset.clear();
-
- for (TopicPartition tp : partitions) {
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(tp);
- if (c == null) {
- futures.add(FutureUtil.failedFuture(
- new IllegalArgumentException("Cannot seek on a partition where we are not subscribed")));
- } else {
- futures.add(c.seekAsync(MessageId.earliest));
- }
- }
-
- FutureUtil.waitForAll(futures).join();
- }
-
- @Override
- public void seekToEnd(TopicPartition... partitions) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- if (partitions.length == 0) {
- partitions = consumers.keySet().toArray(new TopicPartition[0]);
- }
- lastCommittedOffset.clear();
- lastReceivedOffset.clear();
-
- for (TopicPartition tp : partitions) {
- org.apache.pulsar.client.api.Consumer<byte[]> c = consumers.get(tp);
- if (c == null) {
- futures.add(FutureUtil.failedFuture(
- new IllegalArgumentException("Cannot seek on a partition where we are not subscribed")));
- } else {
- futures.add(c.seekAsync(MessageId.latest));
- }
- }
-
- FutureUtil.waitForAll(futures).join();
- }
-
- @Override
- public long position(TopicPartition partition) {
- Long offset = lastReceivedOffset.get(partition);
- if (offset == null && !unpolledPartitions.contains(partition)) {
- return resetOffsets(partition).getValue();
- }
- return unpolledPartitions.contains(partition) ? 0 : offset;
- }
-
- @Override
- public OffsetAndMetadata committed(TopicPartition partition) {
- return lastCommittedOffset.get(partition);
- }
-
- @Override
- public Map<MetricName, ? extends Metric> metrics() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<PartitionInfo> partitionsFor(String s) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<String, List<PartitionInfo>> listTopics() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void pause(TopicPartition... topicPartitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void resume(TopicPartition... topicPartitions) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- try {
- closed = true;
-
- if (isAutoCommit) {
- commitAsync();
- }
-
- client.closeAsync().get(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
- } catch (InterruptedException | ExecutionException | TimeoutException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void wakeup() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * This method is called whenever a new message is received.
- * <p>
- * Messages are guaranteed to be delivered in order and from the same thread for a single consumer
- * <p>
- * This method will only be called once for each message, unless either application or broker crashes.
- * <p>
- * Application is responsible of handling any exception that could be thrown while processing the message.
- *
- * @param consumer the consumer that received the message
- * @param msg
- */
- @Override
- public void received(org.apache.pulsar.client.api.Consumer<byte[]> consumer, Message<byte[]> msg) {
- // Block listener thread if the application is slowing down
- try {
- receivedMessages.put(new QueueItem(consumer, msg));
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- if (closed) {
- // Consumer was closed and the thread was interrupted. Nothing to worry about here
- } else {
- throw new RuntimeException(e);
- }
- }
- }
-
- private SubscriptionInitialPosition resetOffsets(final TopicPartition partition) {
- log.info("Resetting partition {} and seeking to {} position", partition, strategy);
- if (strategy == SubscriptionInitialPosition.Earliest) {
- seekToBeginning(partition);
- } else {
- seekToEnd(partition);
- }
- return strategy;
- }
-
- @SuppressWarnings("unchecked")
- private K getKey(String topic, Message<byte[]> msg) {
- if (!msg.hasKey()) {
- return null;
- }
-
- if (keySchema instanceof PulsarKafkaSchema) {
- PulsarKafkaSchema<K> pulsarKafkaSchema = (PulsarKafkaSchema) keySchema;
- Deserializer<K> kafkaDeserializer = pulsarKafkaSchema.getKafkaDeserializer();
- if (kafkaDeserializer instanceof StringDeserializer) {
- return (K) msg.getKey();
- }
- pulsarKafkaSchema.setTopic(topic);
- }
- // Assume base64 encoding
- byte[] data = Base64.getDecoder().decode(msg.getKey());
- return keySchema.decode(data);
-
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
deleted file mode 100644
index b853459..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/kafka/clients/producer/PulsarKafkaProducer.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.pulsar.client.api.CompressionType;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.kafka.compat.KafkaMessageRouter;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarKafkaSchema;
-import org.apache.pulsar.client.kafka.compat.PulsarProducerKafkaConfig;
-import org.apache.pulsar.client.util.MessageIdUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PulsarKafkaProducer<K, V> implements Producer<K, V> {
-
- private final PulsarClient client;
- private final ProducerBuilder<byte[]> pulsarProducerBuilder;
-
- private final ConcurrentMap<String, org.apache.pulsar.client.api.Producer<byte[]>> producers = new ConcurrentHashMap<>();
-
- private final Schema<K> keySchema;
- private final Schema<V> valueSchema;
-
- private final Partitioner partitioner;
- private volatile Cluster cluster = Cluster.empty();
-
- private final Properties properties;
-
- private static final Logger logger = LoggerFactory.getLogger(PulsarKafkaProducer.class);
-
- public PulsarKafkaProducer(Map<String, Object> configs) {
- this(new ProducerConfig(configs), null, null);
- }
-
- public PulsarKafkaProducer(Map<String, Object> configs, Serializer<K> keySerializer,
- Serializer<V> valueSerializer) {
- this(new ProducerConfig(configs), new PulsarKafkaSchema<>(keySerializer), new PulsarKafkaSchema<>(valueSerializer));
- }
-
- public PulsarKafkaProducer(Map<String, Object> configs, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ProducerConfig(configs), keySchema, valueSchema);
- }
-
- public PulsarKafkaProducer(Properties properties) {
- this(new ProducerConfig(properties), null, null);
- }
-
- public PulsarKafkaProducer(Properties properties, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
- this(new ProducerConfig(properties), new PulsarKafkaSchema<>(keySerializer), new PulsarKafkaSchema<>(valueSerializer));
- }
-
- public PulsarKafkaProducer(Properties properties, Schema<K> keySchema, Schema<V> valueSchema) {
- this(new ProducerConfig(properties), keySchema, valueSchema);
- }
-
- @SuppressWarnings({ "unchecked", "deprecation" })
- private PulsarKafkaProducer(ProducerConfig producerConfig, Schema<K> keySchema, Schema<V> valueSchema) {
-
- if (keySchema == null) {
- Serializer<K> kafkaKeySerializer = producerConfig.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class);
- kafkaKeySerializer.configure(producerConfig.originals(), true);
- this.keySchema = new PulsarKafkaSchema<>(kafkaKeySerializer);
- } else {
- this.keySchema = keySchema;
- producerConfig.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
- }
-
- if (valueSchema == null) {
- Serializer<V> kafkaValueSerializer = producerConfig.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Serializer.class);
- kafkaValueSerializer.configure(producerConfig.originals(), false);
- this.valueSchema = new PulsarKafkaSchema<>(kafkaValueSerializer);
- } else {
- this.valueSchema = valueSchema;
- producerConfig.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
- }
-
- partitioner = producerConfig.getConfiguredInstance(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioner.class);
- partitioner.configure(producerConfig.originals());
-
- this.properties = new Properties();
- producerConfig.originals().forEach(properties::put);
-
- long keepAliveIntervalMs = Long.parseLong(properties.getProperty(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "30000"));
-
- String serviceUrl = producerConfig.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG).get(0);
- try {
- // Support Kafka's ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG in ms.
- // If passed in value is greater than Integer.MAX_VALUE in second will throw IllegalArgumentException.
- int keepAliveInterval = Math.toIntExact(keepAliveIntervalMs / 1000);
- client = PulsarClientKafkaConfig.getClientBuilder(properties)
- .serviceUrl(serviceUrl)
- .keepAliveInterval(keepAliveInterval, TimeUnit.SECONDS)
- .build();
- } catch (ArithmeticException e) {
- String errorMessage = String.format("Invalid value %d for 'connections.max.idle.ms'. " +
- "Please use a value smaller than %d000 milliseconds.", keepAliveIntervalMs, Integer.MAX_VALUE);
- logger.error(errorMessage);
- throw new IllegalArgumentException(errorMessage);
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
-
- pulsarProducerBuilder = PulsarProducerKafkaConfig.getProducerBuilder(client, properties);
-
- // To mimic the same batching mode as Kafka, we need to wait a very little amount of
- // time to batch if the client is trying to send messages fast enough
- long lingerMs = Long.parseLong(properties.getProperty(ProducerConfig.LINGER_MS_CONFIG, "1"));
- pulsarProducerBuilder.batchingMaxPublishDelay(lingerMs, TimeUnit.MILLISECONDS);
-
- String compressionType = properties.getProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG);
- if ("gzip".equals(compressionType)) {
- pulsarProducerBuilder.compressionType(CompressionType.ZLIB);
- } else if ("lz4".equals(compressionType)) {
- pulsarProducerBuilder.compressionType(CompressionType.LZ4);
- }
-
- pulsarProducerBuilder.messageRouter(new KafkaMessageRouter(lingerMs));
-
- int sendTimeoutMillis = Integer.parseInt(properties.getProperty(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "30000"));
- pulsarProducerBuilder.sendTimeout(sendTimeoutMillis, TimeUnit.MILLISECONDS);
-
- // Kafka blocking semantic when blockOnBufferFull=false is different from Pulsar client
- // Pulsar throws error immediately when the queue is full and blockIfQueueFull=false
- // Kafka, on the other hand, still blocks for "max.block.ms" time and then gives error.
- boolean shouldBlockPulsarProducer = sendTimeoutMillis > 0;
- pulsarProducerBuilder.blockIfQueueFull(shouldBlockPulsarProducer);
- }
-
-
- @Override
- public Future<RecordMetadata> send(ProducerRecord<K, V> producerRecord) {
- return send(producerRecord, null);
- }
-
- @Override
- public Future<RecordMetadata> send(ProducerRecord<K, V> producerRecord, Callback callback) {
- org.apache.pulsar.client.api.Producer<byte[]> producer;
-
- try {
- producer = producers.computeIfAbsent(producerRecord.topic(), topic -> createNewProducer(topic));
- } catch (Exception e) {
- if (callback != null) {
- callback.onCompletion(null, e);
- }
- CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
- }
-
- TypedMessageBuilder<byte[]> messageBuilder = buildMessage(producer, producerRecord);
-
- CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
- messageBuilder.sendAsync().thenAccept((messageId) -> {
- future.complete(getRecordMetadata(producerRecord.topic(), messageId));
- }).exceptionally(ex -> {
- future.completeExceptionally(ex);
- return null;
- });
-
- future.handle((recordMetadata, throwable) -> {
- if (callback != null) {
- Exception exception = throwable != null ? new Exception(throwable) : null;
- callback.onCompletion(recordMetadata, exception);
- }
- return null;
- });
-
- return future;
- }
-
- @Override
- public void flush() {
- producers.values().stream()
- .map(p -> p.flushAsync())
- .collect(Collectors.toList())
- .forEach(CompletableFuture::join);
- }
-
- @Override
- public List<PartitionInfo> partitionsFor(String s) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Map<MetricName, ? extends Metric> metrics() {
- return Collections.emptyMap();
- }
-
- @Override
- public void close() {
- close(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
- partitioner.close();
- }
-
- @Override
- public void close(long timeout, TimeUnit unit) {
- try {
- client.closeAsync().get(timeout, unit);
- } catch (InterruptedException | ExecutionException | TimeoutException e) {
- throw new RuntimeException(e);
- }
- }
-
- private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
- try {
- // Add the partitions info for the new topic
- cluster = addPartitionsInfo(cluster, topic);
- return pulsarProducerBuilder.clone()
- .topic(topic)
- .create();
- } catch (PulsarClientException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Add the partitions info for the new topic.
- * Need to ensure the atomicity of the update operation.
- */
- private synchronized Cluster addPartitionsInfo(Cluster cluster, String topic) {
- List<String> partitions = client.getPartitionsForTopic(topic).join();
- // read partitions info
- Set<PartitionInfo> partitionsInfo = new HashSet<>();
- for (int i = 0; i < partitions.size(); i++) {
- partitionsInfo.add(new PartitionInfo(topic, i, null, null, null));
- }
- // create cluster with new partitions info
- Set<PartitionInfo> combinedPartitions = new HashSet<>();
- if (cluster.partitionsForTopic(topic) != null) {
- combinedPartitions.addAll(cluster.partitionsForTopic(topic));
- }
- combinedPartitions.addAll(partitionsInfo);
- return new Cluster(cluster.nodes(), combinedPartitions, new HashSet(cluster.unauthorizedTopics()));
- }
-
- private TypedMessageBuilder<byte[]> buildMessage(org.apache.pulsar.client.api.Producer<byte[]> producer, ProducerRecord<K, V> record) {
- TypedMessageBuilder<byte[]> builder = producer.newMessage();
-
- byte[] keyBytes = null;
- if (record.key() != null) {
- String key = getKey(record.topic(), record.key());
- keyBytes = key.getBytes(StandardCharsets.UTF_8);
- builder.key(key);
- }
-
- if (valueSchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema<V>) valueSchema).setTopic(record.topic());
- }
- byte[] value = valueSchema.encode(record.value());
- builder.value(value);
-
- if (record.partition() != null) {
- // Partition was explicitly set on the record
- builder.property(KafkaMessageRouter.PARTITION_ID, record.partition().toString());
- } else {
- // Get the partition id from the partitioner
- int partition = partitioner.partition(record.topic(), record.key(), keyBytes, record.value(), value, cluster);
- builder.property(KafkaMessageRouter.PARTITION_ID, Integer.toString(partition));
- }
- return builder;
- }
-
- private String getKey(String topic, K key) {
- // If key is a String, we can use it as it is, otherwise, serialize to byte[] and encode in base64
- if (key instanceof String) {
- return (String) key;
- }
- if (keySchema instanceof PulsarKafkaSchema) {
- ((PulsarKafkaSchema) keySchema).setTopic(topic);
- }
- byte[] keyBytes = keySchema.encode(key);
- return Base64.getEncoder().encodeToString(keyBytes);
- }
-
- private RecordMetadata getRecordMetadata(String topic, MessageId messageId) {
- MessageIdImpl msgId = (MessageIdImpl) messageId;
-
- // Combine ledger id and entry id to form offset
- long offset = MessageIdUtils.getOffset(msgId);
- int partition = msgId.getPartitionIndex();
-
- TopicPartition tp = new TopicPartition(topic, partition);
- return new RecordMetadata(tp, offset, 0L);
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java
deleted file mode 100644
index d3fb915..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/KafkaMessageRouter.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.concurrent.ThreadLocalRandom;
-
-import org.apache.pulsar.client.api.HashingScheme;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.TopicMetadata;
-import org.apache.pulsar.client.impl.RoundRobinPartitionMessageRouterImpl;
-
-public class KafkaMessageRouter extends RoundRobinPartitionMessageRouterImpl {
-
- public static final String PARTITION_ID = "pulsar.partition.id";
-
- public KafkaMessageRouter(long maxBatchingDelayMs) {
- super(HashingScheme.JavaStringHash, ThreadLocalRandom.current().nextInt(), true, maxBatchingDelayMs);
- }
-
- @Override
- public int choosePartition(Message<?> msg, TopicMetadata metadata) {
- if (msg.hasProperty(PARTITION_ID)) {
- return Integer.parseInt(msg.getProperty(PARTITION_ID));
- } else {
- return super.choosePartition(msg, metadata);
- }
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java
deleted file mode 100644
index c38d93d..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarClientKafkaConfig.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-
-public class PulsarClientKafkaConfig {
-
- /// Config variables
- public static final String AUTHENTICATION_CLASS = "pulsar.authentication.class";
- public static final String AUTHENTICATION_PARAMS_MAP = "pulsar.authentication.params.map";
- public static final String AUTHENTICATION_PARAMS_STRING = "pulsar.authentication.params.string";
- public static final String USE_TLS = "pulsar.use.tls";
- public static final String TLS_TRUST_CERTS_FILE_PATH = "pulsar.tls.trust.certs.file.path";
- public static final String TLS_ALLOW_INSECURE_CONNECTION = "pulsar.tls.allow.insecure.connection";
- public static final String TLS_HOSTNAME_VERIFICATION = "pulsar.tls.hostname.verification";
-
- public static final String OPERATION_TIMEOUT_MS = "pulsar.operation.timeout.ms";
- public static final String STATS_INTERVAL_SECONDS = "pulsar.stats.interval.seconds";
- public static final String NUM_IO_THREADS = "pulsar.num.io.threads";
-
- public static final String CONNECTIONS_PER_BROKER = "pulsar.connections.per.broker";
-
- public static final String USE_TCP_NODELAY = "pulsar.use.tcp.nodelay";
-
- public static final String CONCURRENT_LOOKUP_REQUESTS = "pulsar.concurrent.lookup.requests";
- public static final String MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION = "pulsar.max.number.rejected.request.per.connection";
-
- public static ClientBuilder getClientBuilder(Properties properties) {
- ClientBuilder clientBuilder = PulsarClient.builder();
-
- if (properties.containsKey(AUTHENTICATION_CLASS)) {
- String className = properties.getProperty(AUTHENTICATION_CLASS);
- try {
- if (properties.containsKey(AUTHENTICATION_PARAMS_STRING)) {
- String authParamsString = (String) properties.get(AUTHENTICATION_PARAMS_STRING);
- clientBuilder.authentication(className, authParamsString);
- } else if (properties.containsKey(AUTHENTICATION_PARAMS_MAP)) {
- Map<String, String> authParams = (Map<String, String>) properties.get(AUTHENTICATION_PARAMS_MAP);
- clientBuilder.authentication(className, authParams);
- } else {
- @SuppressWarnings("unchecked")
- Class<Authentication> clazz = (Class<Authentication>) Class.forName(className);
- Authentication auth = clazz.newInstance();
- clientBuilder.authentication(auth);
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- clientBuilder.enableTls(Boolean.parseBoolean(properties.getProperty(USE_TLS, "false")));
- clientBuilder.allowTlsInsecureConnection(
- Boolean.parseBoolean(properties.getProperty(TLS_ALLOW_INSECURE_CONNECTION, "false")));
- clientBuilder.enableTlsHostnameVerification(
- Boolean.parseBoolean(properties.getProperty(TLS_HOSTNAME_VERIFICATION, "false")));
-
- if (properties.containsKey(TLS_TRUST_CERTS_FILE_PATH)) {
- clientBuilder.tlsTrustCertsFilePath(properties.getProperty(TLS_TRUST_CERTS_FILE_PATH));
- }
-
- if (properties.containsKey(OPERATION_TIMEOUT_MS)) {
- clientBuilder.operationTimeout(Integer.parseInt(properties.getProperty(OPERATION_TIMEOUT_MS)),
- TimeUnit.MILLISECONDS);
- }
-
- if (properties.containsKey(STATS_INTERVAL_SECONDS)) {
- clientBuilder.statsInterval(Integer.parseInt(properties.getProperty(STATS_INTERVAL_SECONDS)),
- TimeUnit.SECONDS);
- }
-
- if (properties.containsKey(NUM_IO_THREADS)) {
- clientBuilder.ioThreads(Integer.parseInt(properties.getProperty(NUM_IO_THREADS)));
- }
-
- if (properties.containsKey(CONNECTIONS_PER_BROKER)) {
- clientBuilder.connectionsPerBroker(Integer.parseInt(properties.getProperty(CONNECTIONS_PER_BROKER)));
- }
-
- if (properties.containsKey(USE_TCP_NODELAY)) {
- clientBuilder.enableTcpNoDelay(Boolean.parseBoolean(properties.getProperty(USE_TCP_NODELAY)));
- }
-
- if (properties.containsKey(CONCURRENT_LOOKUP_REQUESTS)) {
- clientBuilder.maxConcurrentLookupRequests(Integer.parseInt(properties.getProperty(CONCURRENT_LOOKUP_REQUESTS)));
- }
-
- if (properties.containsKey(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)) {
- clientBuilder.maxNumberOfRejectedRequestPerConnection(
- Integer.parseInt(properties.getProperty(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)));
- }
-
- return clientBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java
deleted file mode 100644
index a527827..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarConsumerKafkaConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Arrays;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.RegexSubscriptionMode;
-
-public class PulsarConsumerKafkaConfig {
-
- /// Config variables
- public static final String CONSUMER_NAME = "pulsar.consumer.name";
- public static final String RECEIVER_QUEUE_SIZE = "pulsar.consumer.receiver.queue.size";
- public static final String ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS = "pulsar.consumer.acknowledgments.group.time.millis";
- public static final String TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS = "pulsar.consumer.total.receiver.queue.size.across.partitions";
- public static final String SUBSCRIPTION_TOPICS_MODE = "pulsar.consumer.subscription.topics.mode";
-
- public static ConsumerBuilder<byte[]> getConsumerBuilder(PulsarClient client, Properties properties) {
- ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer();
-
- if (properties.containsKey(CONSUMER_NAME)) {
- consumerBuilder.consumerName(properties.getProperty(CONSUMER_NAME));
- }
-
- if (properties.containsKey(RECEIVER_QUEUE_SIZE)) {
- consumerBuilder.receiverQueueSize(Integer.parseInt(properties.getProperty(RECEIVER_QUEUE_SIZE)));
- }
-
- if (properties.containsKey(TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS)) {
- consumerBuilder.maxTotalReceiverQueueSizeAcrossPartitions(
- Integer.parseInt(properties.getProperty(TOTAL_RECEIVER_QUEUE_SIZE_ACROSS_PARTITIONS)));
- }
-
- if (properties.containsKey(ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS)) {
- consumerBuilder.acknowledgmentGroupTime(
- Long.parseLong(properties.getProperty(ACKNOWLEDGEMENTS_GROUP_TIME_MILLIS)), TimeUnit.MILLISECONDS);
- }
-
- if (properties.containsKey(SUBSCRIPTION_TOPICS_MODE)) {
- RegexSubscriptionMode mode;
- try {
- mode = RegexSubscriptionMode.valueOf(properties.getProperty(SUBSCRIPTION_TOPICS_MODE));
- } catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("Illegal subscription mode, valid values are: "
- + Arrays.asList(RegexSubscriptionMode.values()));
- }
- consumerBuilder.subscriptionTopicsMode(mode);
- }
-
- return consumerBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java
deleted file mode 100644
index aef6dd1..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarKafkaSchema.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-public class PulsarKafkaSchema<T> implements Schema<T> {
-
- private final Serializer<T> kafkaSerializer;
-
- private final Deserializer<T> kafkaDeserializer;
-
- private String topic;
-
- public PulsarKafkaSchema(Serializer<T> serializer) {
- this(serializer, null);
- }
-
- public PulsarKafkaSchema(Deserializer<T> deserializer) {
- this(null, deserializer);
- }
-
- public PulsarKafkaSchema(Serializer<T> serializer, Deserializer<T> deserializer) {
- this.kafkaSerializer = serializer;
- this.kafkaDeserializer = deserializer;
- }
-
- public Serializer<T> getKafkaSerializer() {
- return kafkaSerializer;
- }
-
- public Deserializer<T> getKafkaDeserializer() {
- return kafkaDeserializer;
- }
-
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- @Override
- public byte[] encode(T message) {
- checkArgument(kafkaSerializer != null, "Kafka serializer is not initialized yet");
- return kafkaSerializer.serialize(this.topic, message);
- }
-
- @Override
- public T decode(byte[] message) {
- checkArgument(kafkaDeserializer != null, "Kafka deserializer is not initialized yet");
- return kafkaDeserializer.deserialize(this.topic, message);
- }
-
- @Override
- public SchemaInfo getSchemaInfo() {
- return Schema.BYTES.getSchemaInfo();
- }
-
- @Override
- public Schema<T> clone() {
- return this;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java
deleted file mode 100644
index 5a9a651..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/main/java/org/apache/pulsar/client/kafka/compat/PulsarProducerKafkaConfig.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.client.kafka.compat;
-
-import java.util.Properties;
-
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-
-public class PulsarProducerKafkaConfig {
-
- /// Config variables
- public static final String PRODUCER_NAME = "pulsar.producer.name";
- public static final String INITIAL_SEQUENCE_ID = "pulsar.producer.initial.sequence.id";
-
- public static final String MAX_PENDING_MESSAGES = "pulsar.producer.max.pending.messages";
- public static final String MAX_PENDING_MESSAGES_ACROSS_PARTITIONS = "pulsar.producer.max.pending.messages.across.partitions";
- public static final String BATCHING_ENABLED = "pulsar.producer.batching.enabled";
- public static final String BATCHING_MAX_MESSAGES = "pulsar.producer.batching.max.messages";
-
- public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
- ProducerBuilder<byte[]> producerBuilder = client.newProducer();
-
- if (properties.containsKey(PRODUCER_NAME)) {
- producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
- }
-
- if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
- producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
- }
-
- if (properties.containsKey(MAX_PENDING_MESSAGES)) {
- producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
- }
-
- if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
- producerBuilder.maxPendingMessagesAcrossPartitions(
- Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
- }
-
- producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));
-
- if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
- producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
- }
-
- return producerBuilder;
- }
-}
diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java b/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
deleted file mode 100644
index 4d12418..0000000
--- a/pulsar-client-kafka-compat/pulsar-client-kafka_0_9/src/test/java/org/apache/kafka/clients/producer/PulsarKafkaProducerTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.anyVararg;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-import org.apache.avro.reflect.Nullable;
-import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig;
-import org.apache.pulsar.client.kafka.compat.PulsarProducerKafkaConfig;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.testng.Assert;
-import org.testng.IObjectFactory;
-import org.testng.annotations.ObjectFactory;
-import org.testng.annotations.Test;
-
-@PrepareForTest({PulsarClientKafkaConfig.class, PulsarProducerKafkaConfig.class})
-@PowerMockIgnore({"org.apache.logging.log4j.*", "org.apache.kafka.clients.producer.ProducerInterceptor"})
-public class PulsarKafkaProducerTest {
-
- @Data
- @ToString
- @EqualsAndHashCode
- public static class Foo {
- @Nullable
- private String field1;
- @Nullable
- private String field2;
- private int field3;
- }
-
- @Data
- @ToString
- @EqualsAndHashCode
- public static class Bar {
- private boolean field1;
- }
-
- @ObjectFactory
- // Necessary to make PowerMockito.mockStatic work with TestNG.
- public IObjectFactory getObjectFactory() {
- return new org.powermock.modules.testng.PowerMockObjectFactory();
- }
-
- @Test
- public void testPulsarKafkaProducer() {
- ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
- ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
- doAnswer(invocation -> {
- Assert.assertEquals((int)invocation.getArguments()[0], 1000000, "Send time out is suppose to be 1000.");
- return mockProducerBuilder;
- }).when(mockProducerBuilder).sendTimeout(anyInt(), any(TimeUnit.class));
- doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
- doAnswer(invocation -> {
- Assert.assertEquals((int)invocation.getArguments()[0], 1000, "Keep alive interval is suppose to be 1000.");
- return mockClientBuilder;
- }).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));
-
- PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
- PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
- when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
- when(PulsarProducerKafkaConfig.getProducerBuilder(any(), any())).thenReturn(mockProducerBuilder);
-
- Properties properties = new Properties();
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
-
- new PulsarKafkaProducer<>(properties);
-
- verify(mockClientBuilder, times(1)).keepAliveInterval(1000, TimeUnit.SECONDS);
- verify(mockProducerBuilder, times(1)).sendTimeout(1000000, TimeUnit.MILLISECONDS);
- }
-
- @Test
- public void testPulsarKafkaSendAvro() throws PulsarClientException {
- // Arrange
- PulsarClient mockClient = mock(PulsarClient.class);
- ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
- org.apache.pulsar.client.api.Producer mockProducer = mock(org.apache.pulsar.client.api.Producer.class);
- ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
- CompletableFuture mockPartitionFuture = new CompletableFuture();
- CompletableFuture mockSendAsyncFuture = new CompletableFuture();
- TypedMessageBuilder mockTypedMessageBuilder = mock(TypedMessageBuilderImpl.class);
-
- mockPartitionFuture.complete(new ArrayList<>());
- mockSendAsyncFuture.complete(new MessageIdImpl(1, 1, 1));
- doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
- doReturn(mockClientBuilder).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));
- doReturn(mockClient).when(mockClientBuilder).build();
- doReturn(mockPartitionFuture).when(mockClient).getPartitionsForTopic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).topic(anyString());
- doReturn(mockProducerBuilder).when(mockProducerBuilder).clone();
- doReturn(mockProducer).when(mockProducerBuilder).create();
- doReturn(mockTypedMessageBuilder).when(mockProducer).newMessage();
- doReturn(mockSendAsyncFuture).when(mockTypedMessageBuilder).sendAsync();
- PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
- PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
- when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
- when(PulsarProducerKafkaConfig.getProducerBuilder(any(PulsarClient.class), any(Properties.class))).thenReturn(mockProducerBuilder);
-
- Properties properties = new Properties();
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
- properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
-
- AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
- AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
- // Act
- PulsarKafkaProducer<Foo, Bar> pulsarKafkaProducer = new PulsarKafkaProducer<>(properties, fooSchema, barSchema);
-
- Bar bar = new Bar();
- bar.setField1(true);
-
- Foo foo = new Foo();
- foo.setField1("field1");
- foo.setField2("field2");
- foo.setField3(3);
-
- pulsarKafkaProducer.send(new ProducerRecord<>("topic", 1, foo, bar));
-
- // Verify
- verify(mockTypedMessageBuilder).sendAsync();
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "Invalid value 2147483648000 for 'connections.max.idle.ms'. Please use a value smaller than 2147483647000 milliseconds.")
- public void testPulsarKafkaProducerKeepAliveIntervalIllegalArgumentException() {
- Properties properties = new Properties();
- properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
- properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
- properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
- properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, Long.toString((Integer.MAX_VALUE + 1L) * 1000));
-
- new PulsarKafkaProducer<>(properties);
- }
-
-}
diff --git a/pulsar-flink/README.md b/pulsar-flink/README.md
deleted file mode 100644
index f970a55..0000000
--- a/pulsar-flink/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-# Pulsar Flink Connector
-
-This Pulsar Flink connector is superseded by a newer version of [pulsar-flink](https://flink-packages.org/packages/pulsar-flink-connector)
-in Flink Packages.
-
-Details: https://cwiki.apache.org/confluence/display/FLINK/FLIP-72%3A+Introduce+Pulsar+Connector
\ No newline at end of file
diff --git a/pulsar-flink/pom.xml b/pulsar-flink/pom.xml
deleted file mode 100644
index d82efad..0000000
--- a/pulsar-flink/pom.xml
+++ /dev/null
@@ -1,169 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
- xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar</artifactId>
- <version>2.7.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>pulsar-flink</artifactId>
- <!-- This Pulsar Flink connector is superseded by a new version of `pulsar-flink` connector.
- https://flink-packages.org/packages/pulsar-flink-connector
- https://cwiki.apache.org/confluence/display/FLINK/FLIP-72%3A+Introduce+Pulsar+Connector -->
- <name>Pulsar Flink Connectors (Deprecated)</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- <!-- Projects depending on this project, won't depend on flink-table. -->
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- <version>${flink.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro</artifactId>
- <version>${flink.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>pulsar-client</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-csv</artifactId>
- <version>1.6</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- <version>3.20.0-GA</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- <resource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
-
- <plugins>
- <!-- Generate Test class from avro schema -->
- <plugin>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-maven-plugin</artifactId>
- <version>${avro.version}</version>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>schema</goal>
- </goals>
- <configuration>
- <testSourceDirectory>${project.basedir}/src/test/resources/avro</testSourceDirectory>
- <testOutputDirectory>${project.basedir}/src/test/java/</testOutputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <executions>
- <execution>
- <id>check-style</id>
- <phase>verify</phase>
- <configuration>
- <configLocation>../buildtools/src/main/resources/pulsar/checkstyle.xml</configLocation>
- <suppressionsLocation>../buildtools/src/main/resources/pulsar/suppressions.xml</suppressionsLocation>
- <encoding>UTF-8</encoding>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/pulsar-flink/src/main/java/org/apache/flink/batch/connectors/pulsar/BasePulsarOutputFormat.java b/pulsar-flink/src/main/java/org/apache/flink/batch/connectors/pulsar/BasePulsarOutputFormat.java
deleted file mode 100644
index d061559..0000000
--- a/pulsar-flink/src/main/java/org/apache/flink/batch/connectors/pulsar/BasePulsarOutputFormat.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.flink.batch.connectors.pulsar;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Function;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.flink.api.common.io.RichOutputFormat;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.Preconditions;
-import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.client.impl.conf.ClientConfigurationData;
-import org.apache.pulsar.client.impl.conf.ProducerConfigurationData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
... 7393 lines suppressed ...