You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2022/12/23 12:08:42 UTC
[flink] 01/04: [FLINK-30397][Connector/Pulsar] Remove Pulsar connector
This is an automated email from the ASF dual-hosted git repository.
martijnvisser pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 64305b3c2b047d6231db5e23dec26c0983c10a11
Author: Martijn Visser <ma...@apache.org>
AuthorDate: Tue Dec 13 13:43:13 2022 +0100
[FLINK-30397][Connector/Pulsar] Remove Pulsar connector
---
.../docs/connectors/datastream/pulsar.md | 1024 -----------------
docs/content/docs/connectors/datastream/pulsar.md | 1185 --------------------
.../generated/pulsar_admin_configuration.html | 42 -
.../generated/pulsar_client_configuration.html | 222 ----
.../generated/pulsar_consumer_configuration.html | 162 ---
.../generated/pulsar_producer_configuration.html | 78 --
.../generated/pulsar_sink_configuration.html | 54 -
.../generated/pulsar_source_configuration.html | 66 --
docs/setup_docs.sh | 1 +
.../flink-architecture-tests-production/pom.xml | 5 -
flink-architecture-tests/pom.xml | 7 -
.../3ac3a1dc-681f-4213-9990-b7b3298a20bc | 0
.../f4d91193-72ba-4ce4-ad83-98f780dce581 | 18 -
.../archunit-violations/stored.rules | 4 -
flink-connectors/flink-connector-pulsar/pom.xml | 326 ------
.../pulsar/common/config/PulsarClientFactory.java | 219 ----
.../pulsar/common/config/PulsarConfigBuilder.java | 143 ---
.../common/config/PulsarConfigValidator.java | 105 --
.../pulsar/common/config/PulsarConfiguration.java | 104 --
.../pulsar/common/config/PulsarOptions.java | 529 ---------
.../pulsar/common/metrics/MetricNames.java | 63 --
.../common/metrics/ProducerMetricsInterceptor.java | 67 --
.../pulsar/common/schema/PulsarSchema.java | 236 ----
.../pulsar/common/schema/PulsarSchemaFactory.java | 44 -
.../common/schema/PulsarSchemaTypeInformation.java | 98 --
.../common/schema/PulsarSchemaTypeSerializer.java | 204 ----
.../pulsar/common/schema/PulsarSchemaUtils.java | 207 ----
.../common/schema/factories/AvroSchemaFactory.java | 48 -
.../schema/factories/BaseStructSchemaFactory.java | 42 -
.../common/schema/factories/JSONSchemaFactory.java | 41 -
.../schema/factories/KeyValueSchemaFactory.java | 84 --
.../schema/factories/PrimitiveSchemaFactory.java | 91 --
.../factories/ProtobufNativeSchemaFactory.java | 43 -
.../schema/factories/ProtobufSchemaFactory.java | 43 -
.../schema/factories/StringSchemaFactory.java | 48 -
.../pulsar/common/utils/PulsarExceptionUtils.java | 83 --
.../pulsar/common/utils/PulsarSerdeUtils.java | 161 ---
.../common/utils/PulsarTransactionUtils.java | 70 --
.../flink/connector/pulsar/sink/PulsarSink.java | 145 ---
.../connector/pulsar/sink/PulsarSinkBuilder.java | 410 -------
.../connector/pulsar/sink/PulsarSinkOptions.java | 304 -----
.../pulsar/sink/committer/PulsarCommittable.java | 71 --
.../committer/PulsarCommittableSerializer.java | 65 --
.../pulsar/sink/committer/PulsarCommitter.java | 174 ---
.../pulsar/sink/config/PulsarSinkConfigUtils.java | 117 --
.../pulsar/sink/config/SinkConfiguration.java | 159 ---
.../connector/pulsar/sink/writer/PulsarWriter.java | 260 -----
.../sink/writer/context/PulsarSinkContext.java | 46 -
.../sink/writer/context/PulsarSinkContextImpl.java | 61 -
.../sink/writer/delayer/FixedMessageDelayer.java | 43 -
.../pulsar/sink/writer/delayer/MessageDelayer.java | 62 -
.../pulsar/sink/writer/message/PulsarMessage.java | 111 --
.../sink/writer/message/PulsarMessageBuilder.java | 127 ---
.../sink/writer/router/KeyHashTopicRouter.java | 71 --
.../pulsar/sink/writer/router/MessageKeyHash.java | 85 --
.../sink/writer/router/RoundRobinTopicRouter.java | 65 --
.../pulsar/sink/writer/router/TopicRouter.java | 64 --
.../sink/writer/router/TopicRoutingMode.java | 87 --
.../writer/serializer/PulsarSchemaWrapper.java | 59 -
.../serializer/PulsarSerializationSchema.java | 129 ---
.../PulsarSerializationSchemaWrapper.java | 59 -
.../sink/writer/topic/TopicMetadataListener.java | 181 ---
.../sink/writer/topic/TopicProducerRegister.java | 286 -----
.../connector/pulsar/source/PulsarSource.java | 184 ---
.../pulsar/source/PulsarSourceBuilder.java | 552 ---------
.../pulsar/source/PulsarSourceOptions.java | 624 -----------
.../pulsar/source/config/CursorVerification.java | 49 -
.../source/config/PulsarConsumerBuilder.java | 89 --
.../source/config/PulsarSourceConfigUtils.java | 161 ---
.../pulsar/source/config/SourceConfiguration.java | 257 -----
.../source/enumerator/PulsarSourceEnumState.java | 48 -
.../PulsarSourceEnumStateSerializer.java | 107 --
.../source/enumerator/PulsarSourceEnumerator.java | 290 -----
.../assigner/KeySharedSplitAssigner.java | 93 --
.../assigner/NonSharedSplitAssigner.java | 79 --
.../enumerator/assigner/SharedSplitAssigner.java | 88 --
.../source/enumerator/assigner/SplitAssigner.java | 66 --
.../enumerator/assigner/SplitAssignerBase.java | 140 ---
.../enumerator/assigner/SplitAssignerFactory.java | 62 -
.../source/enumerator/cursor/CursorPosition.java | 92 --
.../source/enumerator/cursor/MessageIdUtils.java | 73 --
.../source/enumerator/cursor/StartCursor.java | 92 --
.../source/enumerator/cursor/StopCursor.java | 156 ---
.../cursor/start/MessageIdStartCursor.java | 64 --
.../cursor/start/TimestampStartCursor.java | 38 -
.../cursor/stop/EventTimestampStopCursor.java | 42 -
.../cursor/stop/LatestMessageStopCursor.java | 58 -
.../cursor/stop/MessageIdStopCursor.java | 55 -
.../enumerator/cursor/stop/NeverStopCursor.java | 33 -
.../cursor/stop/PublishTimestampStopCursor.java | 42 -
.../enumerator/subscriber/PulsarSubscriber.java | 71 --
.../subscriber/impl/BasePulsarSubscriber.java | 91 --
.../subscriber/impl/TopicListSubscriber.java | 85 --
.../subscriber/impl/TopicPatternSubscriber.java | 111 --
.../source/enumerator/topic/TopicMetadata.java | 57 -
.../source/enumerator/topic/TopicNameUtils.java | 137 ---
.../source/enumerator/topic/TopicPartition.java | 156 ---
.../pulsar/source/enumerator/topic/TopicRange.java | 105 --
.../topic/range/FixedKeysRangeGenerator.java | 185 ---
.../topic/range/FixedRangeGenerator.java | 57 -
.../enumerator/topic/range/FullRangeGenerator.java | 48 -
.../enumerator/topic/range/RangeGenerator.java | 104 --
.../topic/range/SplitRangeGenerator.java | 82 --
.../enumerator/topic/range/TopicRangeUtils.java | 136 ---
.../source/reader/PulsarSourceReaderFactory.java | 132 ---
.../deserializer/PulsarDeserializationSchema.java | 137 ---
...DeserializationSchemaInitializationContext.java | 51 -
.../PulsarDeserializationSchemaWrapper.java | 66 --
.../reader/deserializer/PulsarSchemaWrapper.java | 62 -
.../deserializer/PulsarTypeInformationWrapper.java | 66 --
.../source/reader/emitter/PulsarRecordEmitter.java | 94 --
.../reader/fetcher/PulsarFetcherManagerBase.java | 117 --
.../fetcher/PulsarOrderedFetcherManager.java | 75 --
.../fetcher/PulsarUnorderedFetcherManager.java | 71 --
.../reader/source/PulsarOrderedSourceReader.java | 210 ----
.../reader/source/PulsarSourceReaderBase.java | 98 --
.../reader/source/PulsarUnorderedSourceReader.java | 210 ----
.../split/PulsarOrderedPartitionSplitReader.java | 129 ---
.../split/PulsarPartitionSplitReaderBase.java | 306 -----
.../split/PulsarUnorderedPartitionSplitReader.java | 161 ---
.../pulsar/source/split/PulsarPartitionSplit.java | 122 --
.../split/PulsarPartitionSplitSerializer.java | 185 ---
.../source/split/PulsarPartitionSplitState.java | 73 --
.../architecture/TestCodeArchitectureTest.java | 40 -
.../common/config/PulsarConfigBuilderTest.java | 76 --
.../common/config/PulsarConfigValidatorTest.java | 57 -
.../common/config/PulsarConfigurationTest.java | 65 --
.../pulsar/common/schema/PulsarSchemaTest.java | 153 ---
.../schema/PulsarSchemaTypeInformationTest.java | 49 -
.../schema/PulsarSchemaTypeSerializerTest.java | 103 --
.../common/schema/PulsarSchemaUtilsTest.java | 104 --
.../schema/factories/AvroSchemaFactoryTest.java | 214 ----
.../schema/factories/JSONSchemaFactoryTest.java | 71 --
.../factories/KeyValueSchemaFactoryTest.java | 76 --
.../factories/ProtobufNativeSchemaFactoryTest.java | 89 --
.../factories/ProtobufSchemaFactoryTest.java | 69 --
.../pulsar/sink/PulsarSinkBuilderTest.java | 107 --
.../connector/pulsar/sink/PulsarSinkITCase.java | 146 ---
.../committer/PulsarCommittableSerializerTest.java | 53 -
.../pulsar/sink/writer/PulsarWriterTest.java | 194 ----
.../sink/writer/router/KeyHashTopicRouterTest.java | 111 --
.../writer/router/RoundRobinTopicRouterTest.java | 88 --
.../writer/topic/TopicMetadataListenerTest.java | 157 ---
.../writer/topic/TopicProducerRegisterTest.java | 94 --
.../pulsar/source/PulsarSourceBuilderTest.java | 115 --
.../pulsar/source/PulsarSourceITCase.java | 64 --
.../pulsar/source/PulsarUnorderedSourceITCase.java | 60 -
.../PulsarSourceEnumStateSerializerTest.java | 140 ---
.../enumerator/PulsarSourceEnumeratorTest.java | 456 --------
.../assigner/NonSharedSplitAssignerTest.java | 113 --
.../assigner/SharedSplitAssignerTest.java | 121 --
.../enumerator/assigner/SplitAssignerTestBase.java | 127 ---
.../source/enumerator/cursor/StopCursorTest.java | 102 --
.../subscriber/PulsarSubscriberTest.java | 152 ---
.../enumerator/topic/TopicNameUtilsTest.java | 96 --
.../enumerator/topic/TopicPartitionTest.java | 39 -
.../source/enumerator/topic/TopicRangeTest.java | 59 -
.../topic/range/SplitRangeGeneratorTest.java | 82 --
.../topic/range/TopicRangeUtilsTest.java | 84 --
.../PulsarDeserializationSchemaTest.java | 137 ---
.../source/PulsarOrderedSourceReaderTest.java | 212 ----
.../reader/source/PulsarSourceReaderTestBase.java | 241 ----
.../source/PulsarUnorderedSourceReaderTest.java | 27 -
.../PulsarOrderedPartitionSplitReaderTest.java | 91 --
.../split/PulsarPartitionSplitReaderTestBase.java | 417 -------
.../PulsarUnorderedPartitionSplitReaderTest.java | 28 -
.../split/PulsarPartitionSplitSerializerTest.java | 90 --
.../pulsar/testutils/PulsarTestCommonUtils.java | 80 --
.../pulsar/testutils/PulsarTestContext.java | 63 --
.../pulsar/testutils/PulsarTestContextFactory.java | 46 -
.../pulsar/testutils/PulsarTestEnvironment.java | 115 --
.../pulsar/testutils/PulsarTestSuiteBase.java | 65 --
.../connector/pulsar/testutils/SampleData.java | 319 ------
.../pulsar/testutils/extension/SubType.java | 32 -
.../extension/TestOrderlinessExtension.java | 65 --
.../pulsar/testutils/function/ControlSource.java | 267 -----
.../pulsar/testutils/runtime/PulsarRuntime.java | 74 --
.../testutils/runtime/PulsarRuntimeOperator.java | 514 ---------
.../testutils/runtime/PulsarRuntimeUtils.java | 124 --
.../runtime/container/PulsarContainerRuntime.java | 147 ---
.../runtime/mock/BlankBrokerInterceptor.java | 61 -
.../runtime/mock/MockBookKeeperClientFactory.java | 74 --
.../testutils/runtime/mock/MockPulsarService.java | 57 -
.../runtime/mock/NonClosableMockBookKeeper.java | 55 -
.../testutils/runtime/mock/PulsarMockRuntime.java | 114 --
.../mock/SameThreadOrderedSafeExecutor.java | 56 -
.../testutils/sink/PulsarPartitionDataReader.java | 93 --
.../testutils/sink/PulsarSinkTestContext.java | 124 --
.../source/KeyedPulsarPartitionDataWriter.java | 66 --
.../source/PulsarPartitionDataWriter.java | 54 -
.../testutils/source/PulsarSourceTestContext.java | 138 ---
.../source/UnorderedSourceTestSuiteBase.java | 76 --
.../source/cases/KeySharedSubscriptionContext.java | 85 --
.../cases/MultipleTopicConsumingContext.java | 71 --
.../source/cases/SharedSubscriptionContext.java | 46 -
.../source/cases/SingleTopicConsumingContext.java | 73 --
.../src/test/resources/archunit.properties | 31 -
.../src/test/resources/log4j2-test.properties | 35 -
.../test/resources/protobuf/sample_message.proto | 45 -
.../flink-sql-connector-pulsar/pom.xml | 83 --
.../src/main/resources/META-INF/NOTICE | 21 -
.../META-INF/licences/LICENSE.bouncycastle | 7 -
flink-connectors/pom.xml | 2 -
flink-docs/pom.xml | 6 -
.../docs/util/ConfigurationOptionLocator.java | 9 -
.../flink-end-to-end-tests-pulsar/pom.xml | 202 ----
.../flink/tests/util/pulsar/PulsarSinkE2ECase.java | 56 -
.../util/pulsar/PulsarSourceOrderedE2ECase.java | 63 --
.../util/pulsar/PulsarSourceUnorderedE2ECase.java | 63 --
.../FlinkContainerWithPulsarEnvironment.java | 65 --
.../common/PulsarContainerTestEnvironment.java | 31 -
.../source/ExclusiveSubscriptionContext.java | 47 -
.../pulsar/source/FailoverSubscriptionContext.java | 47 -
.../src/test/resources/log4j2-test.properties | 28 -
flink-end-to-end-tests/pom.xml | 1 -
.../org/apache/flink/util/DockerImageVersions.java | 2 -
pom.xml | 1 -
tools/ci/stage.sh | 2 -
tools/maven/suppressions.xml | 4 -
219 files changed, 1 insertion(+), 25805 deletions(-)
diff --git a/docs/content.zh/docs/connectors/datastream/pulsar.md b/docs/content.zh/docs/connectors/datastream/pulsar.md
deleted file mode 100644
index 0778eb8b9eb..00000000000
--- a/docs/content.zh/docs/connectors/datastream/pulsar.md
+++ /dev/null
@@ -1,1024 +0,0 @@
----
-title: Pulsar
-weight: 9
-type: docs
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Apache Pulsar 连接器
-
-Flink 当前提供 [Apache Pulsar](https://pulsar.apache.org) Source 和 Sink 连接器，用户可以使用它从 Pulsar 读取数据，并保证每条数据只被处理一次。
-
-## 添加依赖
-
-Pulsar Source 当前支持 Pulsar 2.8.1 之后的版本，但是 Pulsar Source 使用到了 Pulsar 的[事务机制](https://pulsar.apache.org/docs/zh-CN/txn-what/)，建议在 Pulsar 2.9.2 及其之后的版本上使用 Pulsar Source 进行数据读取。
-
-如果想要了解更多关于 Pulsar API 兼容性设计，可以阅读文档 [PIP-72](https://github.com/apache/pulsar/wiki/PIP-72%3A-Introduce-Pulsar-Interface-Taxonomy%3A-Audience-and-Stability-Classification)。
-
-{{< artifact flink-connector-pulsar >}}
-
-{{< py_download_link "pulsar" >}}
-
-Flink 的流连接器并不会放到发行文件里面一同发布，阅读[此文档]({{< ref "docs/dev/configuration/overview" >}})，了解如何将连接器添加到集群实例内。
-
-## Pulsar Source
-
-{{< hint info >}}
-Pulsar Source 基于 Flink 最新的[批流一体 API]({{< ref "docs/dev/datastream/sources.md" >}}) 进行开发。
-{{< /hint >}}
-
-### 使用示例
-
-Pulsar Source 提供了 builder 类来构造 `PulsarSource` 实例。下面的代码实例使用 builder 类创建的实例会从 “persistent://public/default/my-topic” 的数据开始端进行消费。对应的 Pulsar Source 使用了 **Exclusive**（独占）的订阅方式消费消息，订阅名称为 `my-subscription`，并把消息体的二进制字节流以 UTF-8 的方式编码为字符串。
-
-{{< tabs "pulsar-source-usage" >}}
-{{< tab "Java" >}}
-
-```java
-PulsarSource<String> source = PulsarSource.builder()
- .setServiceUrl(serviceUrl)
- .setAdminUrl(adminUrl)
- .setStartCursor(StartCursor.earliest())
- .setTopics("my-topic")
- .setDeserializationSchema(PulsarDeserializationSchema.flinkSchema(new SimpleStringSchema()))
- .setSubscriptionName("my-subscription")
- .setSubscriptionType(SubscriptionType.Exclusive)
- .build();
-
-env.fromSource(source, WatermarkStrategy.noWatermarks(), "Pulsar Source");
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-pulsar_source = PulsarSource.builder() \
- .set_service_url('pulsar://localhost:6650') \
- .set_admin_url('http://localhost:8080') \
- .set_start_cursor(StartCursor.earliest()) \
- .set_topics("my-topic") \
- .set_deserialization_schema(
- PulsarDeserializationSchema.flink_schema(SimpleStringSchema())) \
- .set_subscription_name('my-subscription') \
- .set_subscription_type(SubscriptionType.Exclusive) \
- .build()
-
-env.from_source(source=pulsar_source,
- watermark_strategy=WatermarkStrategy.for_monotonous_timestamps(),
- source_name="pulsar source")
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-如果使用构造类构造 `PulsarSource`，一定要提供下面几个属性：
-
-- Pulsar 数据消费的地址，使用 `setServiceUrl(String)` 方法提供。
-- Pulsar HTTP 管理地址，使用 `setAdminUrl(String)` 方法提供。
-- Pulsar 订阅名称，使用 `setSubscriptionName(String)` 方法提供。
-- 需要消费的 Topic 或者是 Topic 下面的分区，详见[指定消费的 Topic 或者 Topic 分区](#指定消费的-topic-或者-topic-分区)。
-- 解码 Pulsar 消息的反序列化器，详见[反序列化器](#反序列化器)。
-
-### 指定消费的 Topic 或者 Topic 分区
-
-Pulsar Source 提供了两种订阅 Topic 或 Topic 分区的方式。
-
-- Topic 列表，从这个 Topic 的所有分区上消费消息，例如：
- {{< tabs "pulsar-source-topics" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSource.builder().setTopics("some-topic1", "some-topic2");
-
- // 从 topic "topic-a" 的 0 和 2 分区上消费
- PulsarSource.builder().setTopics("topic-a-partition-0", "topic-a-partition-2");
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSource.builder().set_topics(["some-topic1", "some-topic2"])
-
- # 从 topic "topic-a" 的 0 和 2 分区上消费
- PulsarSource.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2"])
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-- Topic 正则，Pulsar Source 使用给定的正则表达式匹配出所有合规的 Topic，例如：
- {{< tabs "pulsar-source-topic-pattern" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSource.builder().setTopicPattern("topic-*");
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSource.builder().set_topic_pattern("topic-*")
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-#### Topic 名称简写
-
-从 Pulsar 2.0 之后，完整的 Topic 名称格式为 `{persistent|non-persistent}://租户/命名空间/topic`。但是 Pulsar Source 不需要提供 Topic 名称的完整定义，因为 Topic 类型、租户、命名空间都设置了默认值。
-
-| Topic 属性 | 默认值 |
-|:---------|:-------------|
-| Topic 类型 | `persistent` |
-| 租户 | `public` |
-| 命名空间 | `default` |
-
-下面的表格提供了当前 Pulsar Topic 支持的简写方式：
-
-| Topic 名称简写 | 翻译后的 Topic 名称 |
-|:----------------------------------|:-----------------------------------------------|
-| `my-topic` | `persistent://public/default/my-topic` |
-| `my-tenant/my-namespace/my-topic` | `persistent://my-tenant/my-namespace/my-topic` |
-
-{{< hint warning >}}
-对于 Non-persistent（非持久化）Topic，Pulsar Source 不支持简写名称。所以无法将 `non-persistent://public/default/my-topic` 简写成 `non-persistent://my-topic`。
-{{< /hint >}}
-
-#### Pulsar Topic 层次结构
-
-对于 Pulsar 而言，Topic 分区也是一种 Topic。Pulsar 会将一个有分区的 Topic 在内部按照分区的大小拆分成等量的无分区 Topic。
-
-由于 Pulsar 内部的分区实际实现为一个 Topic，我们将用“分区”来指代“仅有一个分区的 Topic（Non-partitioned Topic）”和“具有多个分区的 Topic 下属的分区”。
-
-例如，在 Pulsar 的 `sample` 租户下面的 `flink` 命名空间里面创建了一个有 3 个分区的 Topic，给它起名为 `simple-string`。可以在 Pulsar 上看到如下的 Topic 列表：
-
-| Topic 名称 | 是否分区 |
-|:------------------------------------------------------|:-----|
-| `persistent://sample/flink/simple-string` | 是 |
-| `persistent://sample/flink/simple-string-partition-0` | 否 |
-| `persistent://sample/flink/simple-string-partition-1` | 否 |
-| `persistent://sample/flink/simple-string-partition-2` | 否 |
-
-这意味着，用户可以用上面的子 Topic 去直接消费分区里面的数据，不需要再去基于上层的父 Topic 去消费全部分区的数据。例如：使用 `PulsarSource.builder().setTopics("sample/flink/simple-string-partition-1", "sample/flink/simple-string-partition-2")` 将会只消费 Topic `sample/flink/simple-string` 分区 1 和 2 里面的消息。
-
-#### 配置 Topic 正则表达式
-
-前面提到了 Pulsar Topic 有 `persistent`、`non-persistent` 两种类型，使用正则表达式消费数据的时候，Pulsar Source 会尝试从正则表达式里面解析出消息的类型。例如：`PulsarSource.builder().setTopicPattern("non-persistent://my-topic*")` 会解析出 `non-persistent` 这个 Topic 类型。如果用户使用 Topic 名称简写的方式，Pulsar Source 会使用默认的消息类型 `persistent`。
-
-如果想用正则去消费 `persistent` 和 `non-persistent` 类型的 Topic，需要使用 `RegexSubscriptionMode` 定义 Topic 类型，例如：`setTopicPattern("topic-*", RegexSubscriptionMode.AllTopics)`。
-
-### 反序列化器
-
-反序列化器用于解析 Pulsar 消息，Pulsar Source 使用 `PulsarDeserializationSchema` 来定义反序列化器。用户可以在 builder 类中使用 `setDeserializationSchema(PulsarDeserializationSchema)` 方法配置反序列化器。
-
-如果用户只关心消息体的二进制字节流，并不需要其他属性来解析数据。可以直接使用预定义的 `PulsarDeserializationSchema`。Pulsar Source里面提供了 3 种预定义的反序列化器。
-
-- 使用 Pulsar 的 [Schema](https://pulsar.apache.org/docs/zh-CN/schema-understand/) 解析消息。
- ```java
- // 基础数据类型
- PulsarDeserializationSchema.pulsarSchema(Schema);
-
- // 结构类型 (JSON, Protobuf, Avro, etc.)
- PulsarDeserializationSchema.pulsarSchema(Schema, Class);
-
- // 键值对类型
- PulsarDeserializationSchema.pulsarSchema(Schema, Class, Class);
- ```
-- 使用 Flink 的 `DeserializationSchema` 解析消息。
- {{< tabs "pulsar-deserializer-deserialization-schema" >}}
- {{< tab "Java" >}}
- ```java
- PulsarDeserializationSchema.flinkSchema(DeserializationSchema);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- PulsarDeserializationSchema.flink_schema(DeserializationSchema)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 使用 Flink 的 `TypeInformation` 解析消息。
- {{< tabs "pulsar-deserializer-type-information" >}}
- {{< tab "Java" >}}
- ```java
- PulsarDeserializationSchema.flinkTypeInfo(TypeInformation, ExecutionConfig);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- PulsarDeserializationSchema.flink_type_info(TypeInformation)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-Pulsar 的 `Message<byte[]>` 包含了很多 [额外的属性](https://pulsar.apache.org/docs/zh-CN/concepts-messaging/#%E6%B6%88%E6%81%AF)。例如，消息的 key、消息发送时间、消息生产时间、用户在消息上自定义的键值对属性等。可以使用 `Message<byte[]>` 接口来获取这些属性。
-
-如果用户需要基于这些额外的属性来解析一条消息，可以实现 `PulsarDeserializationSchema` 接口。并一定要确保 `PulsarDeserializationSchema.getProducedType()` 方法返回的 `TypeInformation` 是正确的结果。Flink 使用 `TypeInformation` 将解析出来的结果序列化传递到下游算子。
-
-### Pulsar 订阅
-
-订阅是命名好的配置规则，指导消息如何投递给消费者。Pulsar Source 需要提供一个独立的订阅名称,支持 Pulsar 的四种订阅模式：
-
-- [exclusive（独占）](https://pulsar.apache.org/docs/zh-CN/concepts-messaging/#exclusive)
-- [shared（共享）](https://pulsar.apache.org/docs/zh-CN/concepts-messaging/#shared%E5%85%B1%E4%BA%AB)
-- [failover（灾备）](https://pulsar.apache.org/docs/zh-CN/concepts-messaging/#failover%E7%81%BE%E5%A4%87)
-- [key_shared（key 共享）](https://pulsar.apache.org/docs/zh-CN/concepts-messaging/#key_shared)
-
-当前 Pulsar Source 里，`独占` 和 `灾备` 的实现没有区别，如果 Flink 的一个 reader 挂了，Pulsar Source 会把所有未消费的数据交给其他的 reader 来消费数据。
-
-默认情况下，如果没有指定订阅类型，Pulsar Source 使用共享订阅类型（`SubscriptionType.Shared`）。
-
-{{< tabs "pulsar-subscriptions" >}}
-{{< tab "Java" >}}
-
-```java
-// 名为 "my-shared" 的共享订阅
-PulsarSource.builder().setSubscriptionName("my-shared");
-
-// 名为 "my-exclusive" 的独占订阅
-PulsarSource.builder().setSubscriptionName("my-exclusive").setSubscriptionType(SubscriptionType.Exclusive);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# 名为 "my-shared" 的共享订阅
-PulsarSource.builder().set_subscription_name("my-shared")
-
-# 名为 "my-exclusive" 的独占订阅
-PulsarSource.builder().set_subscription_name("my-exclusive").set_subscription_type(SubscriptionType.Exclusive)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-#### Key_Shared 订阅
-
-当时用 Key_Shared 订阅时，Pulsar 将会基于 Message 的 key 去计算对应的 Hash 值，Hash 取值范围为（0～65535）。我们首先会使用 `Message.getOrderingKey()` 计算 Hash，如果没有则会依次使用 `Message.getKey()` 和 `Message.getKeyBytes()`。对于上述 key 都找不到的消息，我们会使用字符串 `"NO_KEY"` 来计算消息的 Hash 值。
-
-在 Flink Connector 中针对 Key_Shared 订阅提供了两种消费模式，分别是 `KeySharedMode.SPLIT` 和 `KeySharedMode.JOIN`，它们的实际消费行为并不相同。`KeySharedMode.JOIN` 会把所有的给定的 Hash 范围放于一个 Reader 中进行消费，而 `KeySharedMode.SPLIT` 会打散给定的 Hash 范围于不同的 Reader 中消费。
-
-之所以这么设计的主要原因是因为，在 Key_Shared 的订阅模式中，如果一条消息找不到对应的消费者，所有的消息都不会继续往下发送。所以我们提供了 `KeySharedMode.JOIN` 模式，允许用户只消费部分 Hash 范围的消息。
-
-##### 定义 RangeGenerator
-
-如果想在 Pulsar Source 里面使用 `Key_Shared` 订阅，需要提供 `RangeGenerator` 实例。`RangeGenerator` 会生成一组消息 key 的 hash 范围，Pulsar Source 会基于给定的范围来消费数据。
-
-Pulsar Source 也提供了一个名为 `SplitRangeGenerator` 的默认实现，它会基于 flink 数据源的并行度将 hash 范围均分。
-
-由于 Pulsar 并未提供 Key 的 Hash 计算方法，所以我们在 Flink 中提供了名为 `FixedKeysRangeGenerator` 的实现，你可以在 builder 中依次提供需要消费的 Key 内容即可。但需要注意的是，Pulsar 的 Key Hash 值并不对应唯一的一个 Key，所以如果你只想消费某几个 Key 的消息，还需要在后面的代码中使用 `DataStream.filter()` 方法来过滤出对应的消息。
-
-### 起始消费位置
-
-Pulsar Source 使用 `setStartCursor(StartCursor)` 方法给定开始消费的位置。内置的开始消费位置有：
-
-- 从 Topic 里面最早的一条消息开始消费。
- {{< tabs "pulsar-starting-position-earliest" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.earliest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.earliest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 从 Topic 里面最新的一条消息开始消费。
- {{< tabs "pulsar-starting-position-latest" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.latest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.latest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 从给定的消息开始消费。
- {{< tabs "pulsar-starting-position-from-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 与前者不同的是，给定的消息可以跳过，再进行消费。
- {{< tabs "pulsar-starting-position-from-message-id-bool" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageId(MessageId, boolean);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_id(message_id, boolean)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 从给定的消息发布时间开始消费，这个方法因为名称容易导致误解现在已经不建议使用。你可以使用方法 `StartCursor.fromPublishTime(long)`。
- {{< tabs "pulsar-starting-position-message-time" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 从给定的消息发布时间开始消费。
- {{< tabs "pulsar-starting-position-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-{{< hint info >}}
-每条消息都有一个固定的序列号，这个序列号在 Pulsar 上有序排列，其包含了 ledger、entry、partition 等原始信息，用于在 Pulsar 底层存储上查找到具体的消息。
-
-Pulsar 称这个序列号为 `MessageId`，用户可以使用 `DefaultImplementation.newMessageId(long ledgerId, long entryId, int partitionIndex)` 创建它。
-{{< /hint >}}
-
-### 边界
-
-Pulsar Source 默认情况下使用流的方式消费数据。除非任务失败或者被取消，否则将持续消费数据。用户可以使用 `setBoundedStopCursor(StopCursor)` 给定停止消费的位置，这种情况下会使用批的方式进行消费。使用流的方式一样可以给定停止位置，使用 `setUnboundedStopCursor(StopCursor)` 方法即可。
-
-在批模式下，使用 `setBoundedStopCursor(StopCursor)` 来指定一个消费停止位置。
-
-内置的停止消费位置如下：
-
-- 永不停止。
- {{< tabs "pulsar-boundedness-never" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.never();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.never()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于 Pulsar 启动时 Topic 里面最新的那条数据。
- {{< tabs "pulsar-boundedness-latest" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.latest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.latest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某条消息，结果里不包含此消息。
- {{< tabs "pulsar-boundedness-at-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某条消息之后，结果里包含此消息。
- {{< tabs "pulsar-boundedness-after-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某个给定的消息事件时间戳，比如 `Message<byte[]>.getEventTime()`，消费结果里不包含此时间戳的消息。
- {{< tabs "pulsar-boundedness-at-event-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atEventTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_event_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某个给定的消息事件时间戳，比如 `Message<byte[]>.getEventTime()`，消费结果里包含此时间戳的消息。
- {{< tabs "pulsar-boundedness-after-event-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterEventTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_event_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某个给定的消息发布时间戳，比如 `Message<byte[]>.getPublishTime()`，消费结果里不包含此时间戳的消息。
- {{< tabs "pulsar-boundedness-at-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- 停止于某个给定的消息发布时间戳，比如 `Message<byte[]>.getPublishTime()`，消费结果里包含此时间戳的消息。
- {{< tabs "pulsar-boundedness-after-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-### Source 配置项
-
-除了前面提到的配置选项，Pulsar Source 还提供了丰富的选项供 Pulsar 专家使用，在 builder 类里通过 `setConfig(ConfigOption<T>, T)` 和 `setConfig(Configuration)` 方法给定下述的全部配置。
-
-#### Pulsar Java 客户端配置项
-
-Pulsar Source 使用 [Java 客户端](https://pulsar.apache.org/docs/zh-CN/client-libraries-java/)来创建消费实例，相关的配置定义于 Pulsar 的 `ClientConfigurationData` 内。在 `PulsarOptions` 选项中，定义大部分的可供用户定义的配置。
-
-{{< generated/pulsar_client_configuration >}}
-
-#### Pulsar 管理 API 配置项
-
-[管理 API](https://pulsar.apache.org/docs/zh-CN/admin-api-overview/) 用于查询 Topic 的元数据和用正则订阅的时候的 Topic 查找，它与 Java 客户端共享大部分配置。下面列举的配置只供管理 API 使用，`PulsarOptions` 包含了这些配置 。
-
-{{< generated/pulsar_admin_configuration >}}
-
-#### Pulsar 消费者 API 配置项
-
-Pulsar 提供了消费者 API 和读者 API 两套 API 来进行数据消费，它们可用于不同的业务场景。Flink 上的 Pulsar Source 使用消费者 API 进行消费，它的配置定义于 Pulsar 的 `ConsumerConfigurationData` 内。Pulsar Source 将其中大部分的可供用户定义的配置定义于 `PulsarSourceOptions` 内。
-
-{{< generated/pulsar_consumer_configuration >}}
-
-#### Pulsar Source配置项
-
-下述配置主要用于性能调优或者是控制消息确认的行为。如非必要，可以不用强制配置。
-
-{{< generated/pulsar_source_configuration >}}
-
-### 动态分区发现
-
-为了能在启动 Flink 任务之后还能发现在 Pulsar 上扩容的分区或者是新创建的 Topic，Pulsar Source 提供了动态分区发现机制。该机制不需要重启 Flink 任务。对选项 `PulsarSourceOptions.PULSAR_PARTITION_DISCOVERY_INTERVAL_MS` 设置一个正整数即可启用。
-
-{{< tabs "pulsar-dynamic-partition-discovery" >}}
-{{< tab "Java" >}}
-
-```java
-// 10 秒查询一次分区信息
-PulsarSource.builder()
- .setConfig(PulsarSourceOptions.PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, 10000);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# 10 秒查询一次分区信息
-PulsarSource.builder()
- .set_config("pulsar.source.partitionDiscoveryIntervalMs", 10000)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-{{< hint warning >}}
-默认情况下，Pulsar 启用动态分区发现，查询间隔为 30 秒。用户可以给定一个负数，将该功能禁用。如果使用批的方式消费数据，将无法启用该功能。
-{{< /hint >}}
-
-### 事件时间和水位线
-
-默认情况下，Pulsar Source 使用 Pulsar 的 `Message<byte[]>` 里面的时间作为解析结果的时间戳。用户可以使用 `WatermarkStrategy` 来自行解析出想要的消息时间，并向下游传递对应的水位线。
-
-{{< tabs "pulsar-watermarks" >}}
-{{< tab "Java" >}}
-
-```java
-env.fromSource(pulsarSource, new CustomWatermarkStrategy(), "Pulsar Source With Custom Watermark Strategy");
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-env.from_source(pulsar_source, CustomWatermarkStrategy(), "Pulsar Source With Custom Watermark Strategy")
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-[这篇文档]({{< ref "docs/dev/datastream/event-time/generating_watermarks.md" >}})详细讲解了如何定义 `WatermarkStrategy`。
-
-### 消息确认
-
-一旦在 Topic 上创建了订阅，消息便会[存储](https://pulsar.apache.org/docs/zh-CN/concepts-architecture-overview/#%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8)在 Pulsar 里。即使没有消费者，消息也不会被丢弃。只有当 Pulsar Source 同 Pulsar 确认此条消息已经被消费，该消息才以某种机制会被移除。Pulsar Source 支持四种订阅方式，它们的消息确认方式也大不相同。
-
-#### 独占和灾备订阅下的消息确认
-
-`独占` 和 `灾备` 订阅下，Pulsar Source 使用累进式确认方式。确认某条消息已经被处理时，其前面消息会自动被置为已读。Pulsar Source 会在 Flink 完成检查点时将对应时刻消费的消息置为已读，以此来保证 Pulsar 状态与 Flink 状态一致。
-
-如果用户没有在 Flink 上启用检查点，Pulsar Source 可以使用周期性提交来将消费状态提交给 Pulsar，使用配置 `PulsarSourceOptions.PULSAR_AUTO_COMMIT_CURSOR_INTERVAL` 来进行定义。
-
-需要注意的是，此种场景下，Pulsar Source 并不依赖于提交到 Pulsar 的状态来做容错。消息确认只是为了能在 Pulsar 端看到对应的消费处理情况。
-
-#### 共享和 key 共享订阅下的消息确认
-
-`共享` 和 `key 共享` 需要依次确认每一条消息，所以 Pulsar Source 在 Pulsar 事务里面进行消息确认，然后将事务提交到 Pulsar。
-
-首先需要在 Pulsar 的 `borker.conf` 文件里面启用事务：
-
-```text
-transactionCoordinatorEnabled=true
-```
-
-Pulsar Source 创建的事务的默认超时时间为 3 小时，请确保这个时间大于 Flink 检查点的间隔。用户可以使用 `PulsarSourceOptions.PULSAR_TRANSACTION_TIMEOUT_MILLIS` 来设置事务的超时时间。
-
-如果用户无法启用 Pulsar 的事务，或者是因为项目禁用了检查点，需要将 `PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE` 选项设置为 `true`，消息从 Pulsar 消费后会被立刻置为已读。Pulsar Source 无法保证此种场景下的消息一致性。
-
-Pulsar Source 在 Pulsar 上使用日志的形式记录某个事务下的消息确认，为了更好的性能，请缩短 Flink 做检查点的间隔。
-
-## Pulsar Sink
-
-Pulsar Sink 连接器可以将经过 Flink 处理后的数据写入一个或多个 Pulsar Topic 或者 Topic 下的某些分区。
-
-{{< hint info >}}
-Pulsar Sink 基于 Flink 最新的 [Sink API](https://cwiki.apache.org/confluence/display/FLINK/FLIP-191%3A+Extend+unified+Sink+interface+to+support+small+file+compaction) 实现。
-
-如果想要使用旧版的使用 `SinkFuntion` 接口实现的 Sink 连接器，可以使用 StreamNative 维护的 [pulsar-flink](https://github.com/streamnative/pulsar-flink)。
-{{< /hint >}}
-
-### 使用示例
-
-Pulsar Sink 使用 builder 类来创建 `PulsarSink` 实例。
-
-下面示例展示了如何通过 Pulsar Sink 以“至少一次”的语义将字符串类型的数据发送给 topic1。
-
-{{< tabs "46e225b1-1e34-4ff3-890c-aa06a2b99c0a" >}}
-{{< tab "Java" >}}
-
-```java
-DataStream<String> stream = ...
-
-PulsarSink<String> sink = PulsarSink.builder()
- .setServiceUrl(serviceUrl)
- .setAdminUrl(adminUrl)
- .setTopics("topic1")
- .setSerializationSchema(PulsarSerializationSchema.flinkSchema(new SimpleStringSchema()))
- .setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
- .build();
-
-stream.sinkTo(sink);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-stream = ...
-
-pulsar_sink = PulsarSink.builder() \
- .set_service_url('pulsar://localhost:6650') \
- .set_admin_url('http://localhost:8080') \
- .set_topics("topic1") \
- .set_serialization_schema(PulsarSerializationSchema.flink_schema(SimpleStringSchema())) \
- .set_delivery_guarantee(DeliveryGuarantee.AT_LEAST_ONCE) \
- .build()
-
-stream.sink_to(pulsar_sink)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-下列为创建一个 `PulsarSink` 实例必需的属性：
-
-- Pulsar 数据消费的地址，使用 `setServiceUrl(String)` 方法提供。
-- Pulsar HTTP 管理地址，使用 `setAdminUrl(String)` 方法提供。
-- 需要发送到的 Topic 或者是 Topic 下面的分区，详见[指定写入的topic或者topic分区](#指定写入的topic或者topic分区)。
-- 编码 Pulsar 消息的序列化器，详见[序列化器](#序列化器)。
-
-在创建 `PulsarSink` 时，建议使用 `setProducerName(String)` 来指定 `PulsarSink` 内部使用的 Pulsar 生产者名称。这样方便在数据监控页面找到对应的生产者监控指标。
-
-### 指定写入的 Topic 或者 Topic 分区
-
-`PulsarSink` 指定写入 Topic 的方式和 Pulsar Source [指定消费的 Topic 或者 Topic 分区](#指定消费的-topic-或者-topic-分区)的方式类似。`PulsarSink` 支持以 mixin 风格指定写入的 Topic 或分区。因此，可以指定一组 Topic 或者分区或者是两者都有。
-
-{{< tabs "3d452e6b-bffd-42f7-bb91-974b306262ca" >}}
-{{< tab "Java" >}}
-
-```java
-// Topic "some-topic1" 和 "some-topic2"
-PulsarSink.builder().setTopics("some-topic1", "some-topic2")
-
-// Topic "topic-a" 的分区 0 和 2
-PulsarSink.builder().setTopics("topic-a-partition-0", "topic-a-partition-2")
-
-// Topic "topic-a" 以及 Topic "some-topic2" 分区 0 和 2
-PulsarSink.builder().setTopics("topic-a-partition-0", "topic-a-partition-2", "some-topic2")
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# Topic "some-topic1" 和 "some-topic2"
-PulsarSink.builder().set_topics(["some-topic1", "some-topic2"])
-
-# Topic "topic-a" 的分区 0 和 2
-PulsarSink.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2"])
-
-# Topic "topic-a" 以及 Topic "some-topic2" 分区 0 和 2
-PulsarSink.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2", "some-topic2"])
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-动态分区发现默认处于开启状态，这意味着 `PulsarSink` 将会周期性地从 Pulsar 集群中查询 Topic 的元数据来获取可能有的分区数量变更信息。使用 `PulsarSinkOptions.PULSAR_TOPIC_METADATA_REFRESH_INTERVAL` 配置项来指定查询的间隔时间。
-
-可以选择实现 `TopicRouter` 接口来自定义[消息路由策略](#消息路由策略)。此外，阅读 [Topic 名称简写](#topic-名称简写)将有助于理解 Pulsar 的分区在 Pulsar 连接器中的配置方式。
-
-{{< hint warning >}}
-如果在 `PulsarSink` 中同时指定了某个 Topic 和其下属的分区，那么 `PulsarSink` 将会自动将两者合并，仅使用外层的 Topic。
-
-举个例子，如果通过 `PulsarSink.builder().setTopics("some-topic1", "some-topic1-partition-0")` 来指定写入的 Topic，那么其结果等价于 `PulsarSink.builder().setTopics("some-topic1")`。
-{{< /hint >}}
-
-### 序列化器
-
-序列化器（`PulsarSerializationSchema`）负责将 Flink 中的每条记录序列化成 byte 数组，并通过网络发送至指定的写入 Topic。和 Pulsar Source 类似的是，序列化器同时支持使用基于 Flink 的 `SerializationSchema` 接口实现序列化器和使用 Pulsar 原生的 `Schema` 类型实现的序列化器。不过序列化器并不支持 Pulsar 的 `Schema.AUTO_PRODUCE_BYTES()`。
-
-如果不需要指定 [Message](https://pulsar.apache.org/api/client/2.9.0-SNAPSHOT/org/apache/pulsar/client/api/Message.html) 接口中提供的 key 或者其他的消息属性，可以从上述 2 种预定义的 `PulsarSerializationSchema` 实现中选择适合需求的一种使用。
-
-- 使用 Pulsar 的 [Schema](https://pulsar.apache.org/docs/zh-CN/schema-understand/) 来序列化 Flink 中的数据。
- ```java
- // 原始数据类型
- PulsarSerializationSchema.pulsarSchema(Schema)
-
- // 有结构数据类型（JSON、Protobuf、Avro 等）
- PulsarSerializationSchema.pulsarSchema(Schema, Class)
-
- // 键值对类型
- PulsarSerializationSchema.pulsarSchema(Schema, Class, Class)
- ```
-- 使用 Flink 的 `SerializationSchema` 来序列化数据。
-
- {{< tabs "b65b9978-b1d6-4b0d-ade8-78098e0f23d8" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSerializationSchema.flinkSchema(SerializationSchema)
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSerializationSchema.flink_schema(SimpleStringSchema())
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-同时使用 `PulsarSerializationSchema.pulsarSchema()` 以及在 builder 中指定 `PulsarSinkBuilder.enableSchemaEvolution()` 可以启用 [Schema evolution](https://pulsar.apache.org/docs/zh-CN/schema-evolution-compatibility/#schema-evolution) 特性。该特性会使用 Pulsar Broker 端提供的 Schema 版本兼容性检测以及 Schema 版本演进。下列示例展示了如何启用 Schema Evolution。
-
-```java
-Schema<SomePojo> schema = Schema.AVRO(SomePojo.class);
-PulsarSerializationSchema<SomePojo> pulsarSchema = PulsarSerializationSchema.pulsarSchema(schema, SomePojo.class);
-
-PulsarSink<String> sink = PulsarSink.builder()
- ...
- .setSerializationSchema(pulsarSchema)
- .enableSchemaEvolution()
- .build();
-```
-
-{{< hint warning >}}
-如果想要使用 Pulsar 原生的 Schema 序列化消息而不需要 Schema Evolution 特性，那么写入的 Topic 会使用 `Schema.BYTES` 作为消息的 Schema，对应 Topic 的消费者需要自己负责反序列化的工作。
-
-例如，如果使用 `PulsarSerializationSchema.pulsarSchema(Schema.STRING)` 而不使用 `PulsarSinkBuilder.enableSchemaEvolution()`。那么在写入 Topic 中所记录的消息 Schema 将会是 `Schema.BYTES`。
-{{< /hint >}}
-
-### 消息路由策略
-
-在 Pulsar Sink 中，消息路由发生在于分区之间，而非上层 Topic。对于给定 Topic 的情况，路由算法会首先会查询出 Topic 之上所有的分区信息，并在这些分区上实现消息的路由。Pulsar Sink 默认提供 2 种路由策略的实现。
-
-- `KeyHashTopicRouter`：使用消息的 key 对应的哈希值来取模计算出消息对应的 Topic 分区。
-
- 使用此路由可以将具有相同 key 的消息发送至同一个 Topic 分区。消息的 key 可以在自定义 `PulsarSerializationSchema` 时，在 `serialize()` 方法内使用 `PulsarMessageBuilder.key(String key)` 来予以指定。
-
- 如果消息没有包含 key，此路由策略将从 Topic 分区中随机选择一个发送。
-
- 可以使用 `MessageKeyHash.JAVA_HASH` 或者 `MessageKeyHash.MURMUR3_32_HASH` 两种不同的哈希算法来计算消息 key 的哈希值。使用 `PulsarSinkOptions.PULSAR_MESSAGE_KEY_HASH` 配置项来指定想要的哈希算法。
-
-- `RoundRobinRouter`：轮换使用用户给定的 Topic 分区。
-
- 消息将会轮替地选取 Topic 分区，当往某个 Topic 分区里写入指定数量的消息后，将会轮换至下一个 Topic 分区。使用 `PulsarSinkOptions.PULSAR_BATCHING_MAX_MESSAGES` 指定向一个 Topic 分区中写入的消息数量。
-
-还可以通过实现 `TopicRouter` 接口来自定义消息路由策略，请注意 TopicRouter 的实现需要能被序列化。
-
-在 `TopicRouter` 内可以指定任意的 Topic 分区（即使这个 Topic 分区不在 `setTopics()` 指定的列表中）。因此，当使用自定义的 `TopicRouter` 时，`PulsarSinkBuilder.setTopics` 选项是可选的。
-
-```java
-@PublicEvolving
-public interface TopicRouter<IN> extends Serializable {
-
- String route(IN in, List<String> partitions, PulsarSinkContext context);
-
- default void open(SinkConfiguration sinkConfiguration) {
- // 默认无操作
- }
-}
-```
-
-{{< hint info >}}
-如前文所述，Pulsar 分区的内部被实现为一个无分区的 Topic，一般情况下 Pulsar 客户端会隐藏这个实现，并且提供内置的消息路由策略。Pulsar Sink 并没有使用 Pulsar 客户端提供的路由策略和封装，而是使用了 Pulsar 客户端更底层的 API 自行实现了消息路由逻辑。这样做的主要目的是能够在属于不同 Topic 的分区之间定义更灵活的消息路由策略。
-
-详情请参考 Pulsar 的 [partitioned topics](https://pulsar.apache.org/docs/zh-CN/cookbooks-partitioned/)。
-{{< /hint >}}
-
-### 发送一致性
-
-`PulsarSink` 支持三种发送一致性。
-
-- `NONE`：Flink 应用运行时可能出现数据丢失的情况。在这种模式下，Pulsar Sink 发送消息后并不会检查消息是否发送成功。此模式具有最高的吞吐量，可用于一致性没有要求的场景。
-- `AT_LEAST_ONCE`：每条消息**至少有**一条对应消息发送至 Pulsar，发送至 Pulsar 的消息可能会因为 Flink 应用重启而出现重复。
-- `EXACTLY_ONCE`：每条消息**有且仅有**一条对应消息发送至 Pulsar。发送至 Pulsar 的消息不会有重复也不会丢失。Pulsar Sink 内部依赖 [Pulsar 事务](https://pulsar.apache.org/docs/zh-CN/transactions/)和两阶段提交协议来保证每条记录都能正确发往 Pulsar。
-
-### 消息延时发送
-
-[消息延时发送](https://pulsar.apache.org/docs/zh-CN/next/concepts-messaging/#%E6%B6%88%E6%81%AF%E5%BB%B6%E8%BF%9F%E4%BC%A0%E9%80%92)特性可以让指定发送的每一条消息需要延时一段时间后才能被下游的消费者所消费。当延时消息发送特性启用时，Pulsar Sink 会**立刻**将消息发送至 Pulsar Broker。但该消息在指定的延迟时间到达前将会保持对下游消费者不可见。
-
-消息延时发送仅在 `Shared` 订阅模式下有效，在 `Exclusive` 和 `Failover` 模式下该特性无效。
-
-可以使用 `MessageDelayer.fixed(Duration)` 创建一个 `MessageDelayer` 来为所有消息指定恒定的接收时延，或者实现 `MessageDelayer` 接口来为不同的消息指定不同的接收时延。
-
-{{< hint warning >}}
-消息对下游消费者的可见时间应当基于 `PulsarSinkContext.processTime() `计算得到。
-{{< /hint >}}
-
-### Sink 配置项
-
-可以在 builder 类里通过 `setConfig(ConfigOption<T>, T)` 和 `setConfig(Configuration)` 方法给定下述的全部配置。
-
-#### PulsarClient 和 PulsarAdmin 配置项
-
-Pulsar Sink 和 Pulsar Source 公用的配置选项可参考
-
-- [Pulsar Java 客户端配置项](#pulsar-java-客户端配置项)
-- [Pulsar 管理 API 配置项](#pulsar-管理-API-配置项)
-
-#### Pulsar 生产者 API 配置项
-
-Pulsar Sink 使用生产者 API 来发送消息。Pulsar 的 `ProducerConfigurationData` 中大部分的配置项被映射为 `PulsarSinkOptions` 里的选项。
-
-{{< generated/pulsar_producer_configuration >}}
-
-#### Pulsar Sink 配置项
-
-下述配置主要用于性能调优或者是控制消息确认的行为。如非必要，可以不用考虑配置。
-
-{{< generated/pulsar_sink_configuration >}}
-
-### Sink 监控指标
-
-下列表格列出了当前 Sink 支持的监控指标，前 6 个指标是 [FLIP-33: Standardize Connector Metrics]([https://cwiki.apache.org/confluence/display/FLINK/FLIP-33%3A+Standardize+Connector+Metrics](https://cwiki.apache.org/confluence/display/FLINK/FLIP-33%3A+Standardize+Connector+Metrics)) 中规定的 Sink 连接器应当支持的标准指标。
-
-<table class="table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 15%">Scope</th>
- <th class="text-left" style="width: 18%">Metrics</th>
- <th class="text-left" style="width: 18%">User Variables</th>
- <th class="text-left" style="width: 39%">Description</th>
- <th class="text-left" style="width: 10%">Type</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <th rowspan="13">Operator</th>
- <td>numBytesOut</td>
- <td>n/a</td>
- <td>Pulsar Sink 启动后总共发出的字节数</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>numBytesOutPerSecond</td>
- <td>n/a</td>
- <td>每秒发送的字节数</td>
- <td>Meter</td>
- </tr>
- <tr>
- <td>numRecordsOut</td>
- <td>n/a</td>
- <td>Pulsar Sink 启动后总共发出的消息数</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>numRecordsOutPerSecond</td>
- <td>n/a</td>
- <td>每秒发送的消息数</td>
- <td>Meter</td>
- </tr>
- <tr>
- <td>numRecordsOutErrors</td>
- <td>n/a</td>
- <td>总共发送消息失败的次数</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>currentSendTime</td>
- <td>n/a</td>
- <td>最近一条消息从被放入客户端缓冲队列到收到消息确认的时间</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.numAcksReceived</td>
- <td>n/a</td>
- <td>总共收到的确认数</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>PulsarSink.sendLatencyMax</td>
- <td>n/a</td>
- <td>所有生产者的最大发送延迟</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency50Pct</td>
- <td>ProducerName</td>
- <td>某个生产者在过去的一个窗口内的发送延迟的中位数</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency75Pct</td>
- <td>ProducerName</td>
- <td>某个生产者在过去的一个窗口内的发送延迟的 75 百分位数</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency95Pct</td>
- <td>ProducerName</td>
- <td>某个生产者在过去的一个窗口内的发送延迟的 95 百分位数</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency99Pct</td>
- <td>ProducerName</td>
- <td>某个生产者在过去的一个窗口内的发送延迟的 99 百分位数</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency999Pct</td>
- <td>ProducerName</td>
- <td>某个生产者在过去的一个窗口内的发送延迟的 99.9 百分位数</td>
- <td>Gauge</td>
- </tr>
- </tbody>
-</table>
-
-{{< hint info >}}
-指标 `numBytesOut`、`numRecordsOut` 和 `numRecordsOutErrors` 从 Pulsar Producer 实例的监控指标中获得。
-
-`currentSendTime` 记录了最近一条消息从放入生产者的缓冲队列到消息被消费确认所耗费的时间。这项指标在 `NONE` 发送一致性下不可用。
-{{< /hint >}}
-
-默认情况下，Pulsar 生产者每隔 60 秒才会刷新一次监控数据，然而 Pulsar Sink 每 500 毫秒就会从 Pulsar 生产者中获得最新的监控数据。因此 `numRecordsOut`、`numBytesOut`、`numAcksReceived` 以及 `numRecordsOutErrors` 4 个指标实际上每 60 秒才会刷新一次。
-
-如果想要更高地刷新评率，可以通过如下方式来将 Pulsar 生产者的监控数据刷新频率调整至相应值（最低为1s）：
-{{< tabs "b65b9978-b1d6-4b0d-ade8-78098e0f23d1" >}}
-
-{{< tab "Java" >}}
-```java
-builder.setConfig(PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS, 1L);
-```
-{{< /tab >}}
-
-{{< tab "Python" >}}
-```python
-builder.set_config("pulsar.client.statsIntervalSeconds", "1")
-```
-{{< /tab >}}
-
-{{< /tabs >}}
-
-`numBytesOutRate` 和 `numRecordsOutRate` 指标是 Flink 内部通过 `numBytesOut` 和 `numRecordsOut` 计数器，在一个 60 秒的窗口内计算得到的。
-
-### 设计思想简述
-
-Pulsar Sink 遵循 [FLIP-191](https://cwiki.apache.org/confluence/display/FLINK/FLIP-191%3A+Extend+unified+Sink+interface+to+support+small+file+compaction) 中定义的 Sink API 设计。
-
-#### 无状态的 SinkWriter
-
-在 `EXACTLY_ONCE` 一致性下，Pulsar Sink 不会将事务相关的信息存放于检查点快照中。这意味着当 Flink 应用重启时，Pulsar Sink 会创建新的事务实例。上一次运行过程中任何未提交事务中的消息会因为超时中止而无法被下游的消费者所消费。这样的设计保证了 SinkWriter 是无状态的。
-
-#### Pulsar Schema Evolution
-
-[Pulsar Schema Evolution](https://pulsar.apache.org/docs/zh-CN/schema-evolution-compatibility/) 允许用户在一个 Flink 应用程序中使用的数据模型发生特定改变后（比如向基于 ARVO 的 POJO 类中增加或删除一个字段），仍能使用同一个 Flink 应用程序的代码。
-
-可以在 Pulsar 集群内指定哪些类型的数据模型的改变是被允许的，详情请参阅 [Pulsar Schema Evolution](https://pulsar.apache.org/docs/zh-CN/schema-evolution-compatibility/)。
-
-## 升级至最新的连接器
-
-常见的升级步骤，请参阅[升级应用程序和 Flink 版本]({{< ref "docs/ops/upgrading" >}})。Pulsar 连接器没有在 Flink 端存储消费的状态，所有的消费信息都推送到了 Pulsar。所以需要注意下面的事项：
-
-* 不要同时升级 Pulsar 连接器和 Pulsar 服务端的版本。
-* 使用最新版本的 Pulsar 客户端来消费消息。
-
-## 问题诊断
-
-使用 Flink 和 Pulsar 交互时如果遇到问题，由于 Flink 内部实现只是基于 Pulsar 的 [Java 客户端](https://pulsar.apache.org/docs/zh-CN/client-libraries-java/)和[管理 API](https://pulsar.apache.org/docs/zh-CN/admin-api-overview/) 而开发的。
-
-用户遇到的问题可能与 Flink 无关，请先升级 Pulsar 的版本、Pulsar 客户端的版本，或者修改 Pulsar 的配置、Pulsar 连接器的配置来尝试解决问题。
-
-## 已知问题
-
-本节介绍有关 Pulsar 连接器的一些已知问题。
-
-### 在 Java 11 上使用不稳定
-
-Pulsar connector 在 Java 11 中有一些尚未修复的问题。我们当前推荐在 Java 8 环境中运行Pulsar connector.
-
-### 不自动重连，而是抛出TransactionCoordinatorNotFound异常
-
-Pulsar 事务机制仍在积极发展中，当前版本并不稳定。 Pulsar 2.9.2
-引入了这个问题 [a break change](https://github.com/apache/pulsar/pull/13135)。
-如果您使用 Pulsar 2.9.2或更高版本与较旧的 Pulsar 客户端一起使用，您可能会收到一个“TransactionCoordinatorNotFound”异常。
-
-您可以使用最新的`pulsar-client-all`分支来解决这个问题。
-
-{{< top >}}
diff --git a/docs/content/docs/connectors/datastream/pulsar.md b/docs/content/docs/connectors/datastream/pulsar.md
deleted file mode 100644
index 34a3bbcd762..00000000000
--- a/docs/content/docs/connectors/datastream/pulsar.md
+++ /dev/null
@@ -1,1185 +0,0 @@
----
-title: Pulsar
-weight: 9
-type: docs
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-# Apache Pulsar Connector
-
-Flink provides an [Apache Pulsar](https://pulsar.apache.org) connector for reading and writing data from and to Pulsar topics with exactly-once guarantees.
-
-## Dependency
-
-You can use the connector with the Pulsar 2.8.1 or higher. Because the Pulsar connector supports
-Pulsar [transactions](https://pulsar.apache.org/docs/en/txn-what/), it is recommended to use the Pulsar 2.9.2 or higher.
-Details on Pulsar compatibility can be found in [PIP-72](https://github.com/apache/pulsar/wiki/PIP-72%3A-Introduce-Pulsar-Interface-Taxonomy%3A-Audience-and-Stability-Classification).
-
-{{< artifact flink-connector-pulsar >}}
-
-{{< py_download_link "pulsar" >}}
-
-Flink's streaming connectors are not part of the binary distribution.
-See how to link with them for cluster execution [here]({{< ref "docs/dev/configuration/overview" >}}).
-
-## Pulsar Source
-
-{{< hint info >}}
-This part describes the Pulsar source based on the new
-[data source]({{< ref "docs/dev/datastream/sources.md" >}}) API.
-{{< /hint >}}
-
-### Usage
-
-The Pulsar source provides a builder class for constructing a PulsarSource instance. The code snippet below builds a PulsarSource instance. It consumes messages from the earliest cursor of the topic
-"persistent://public/default/my-topic" in **Exclusive** subscription type (`my-subscription`)
-and deserializes the raw payload of the messages as strings.
-
-{{< tabs "pulsar-source-usage" >}}
-{{< tab "Java" >}}
-
-```java
-PulsarSource<String> source = PulsarSource.builder()
- .setServiceUrl(serviceUrl)
- .setAdminUrl(adminUrl)
- .setStartCursor(StartCursor.earliest())
- .setTopics("my-topic")
- .setDeserializationSchema(PulsarDeserializationSchema.flinkSchema(new SimpleStringSchema()))
- .setSubscriptionName("my-subscription")
- .setSubscriptionType(SubscriptionType.Exclusive)
- .build();
-
-env.fromSource(source, WatermarkStrategy.noWatermarks(), "Pulsar Source");
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-pulsar_source = PulsarSource.builder() \
- .set_service_url('pulsar://localhost:6650') \
- .set_admin_url('http://localhost:8080') \
- .set_start_cursor(StartCursor.earliest()) \
- .set_topics("my-topic") \
- .set_deserialization_schema(
- PulsarDeserializationSchema.flink_schema(SimpleStringSchema())) \
- .set_subscription_name('my-subscription') \
- .set_subscription_type(SubscriptionType.Exclusive) \
- .build()
-
-env.from_source(source=pulsar_source,
- watermark_strategy=WatermarkStrategy.for_monotonous_timestamps(),
- source_name="pulsar source")
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-The following properties are **required** for building a PulsarSource:
-
-- Pulsar service URL, configured by `setServiceUrl(String)`
-- Pulsar service HTTP URL (also known as admin URL), configured by `setAdminUrl(String)`
-- Pulsar subscription name, configured by `setSubscriptionName(String)`
-- Topics / partitions to subscribe, see the following
- [topic-partition subscription](#topic-partition-subscription) for more details.
-- Deserializer to parse Pulsar messages, see the following
- [deserializer](#deserializer) for more details.
-
-It is recommended to set the consumer name in Pulsar Source by `setConsumerName(String)`.
-This sets a unique name for the Flink connector in the Pulsar statistic dashboard.
-You can use it to monitor the performance of your Flink connector and applications.
-
-### Topic-partition Subscription
-
-Pulsar source provide two ways of topic-partition subscription:
-
-- Topic list, subscribing messages from all partitions in a list of topics. For example:
- {{< tabs "pulsar-source-topics" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSource.builder().setTopics("some-topic1", "some-topic2");
-
- // Partition 0 and 2 of topic "topic-a"
- PulsarSource.builder().setTopics("topic-a-partition-0", "topic-a-partition-2");
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSource.builder().set_topics(["some-topic1", "some-topic2"])
-
- # Partition 0 and 2 of topic "topic-a"
- PulsarSource.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2"])
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-- Topic pattern, subscribing messages from all topics whose name matches the provided regular expression. For example:
- {{< tabs "pulsar-source-topic-pattern" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSource.builder().setTopicPattern("topic-*");
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSource.builder().set_topic_pattern("topic-*")
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-#### Flexible Topic Naming
-
-Since Pulsar 2.0, all topic names internally are in a form of `{persistent|non-persistent}://tenant/namespace/topic`.
-Now, for partitioned topics, you can use short names in many cases (for the sake of simplicity).
-The flexible naming system stems from the fact that there is now a default topic type, tenant, and namespace in a Pulsar cluster.
-
-| Topic property | Default |
-|:---------------|:-------------|
-| topic type | `persistent` |
-| tenant | `public` |
-| namespace | `default` |
-
-This table lists a mapping relationship between your input topic name and the translated topic name:
-
-| Input topic name | Translated topic name |
-|:----------------------------------|:-----------------------------------------------|
-| `my-topic` | `persistent://public/default/my-topic` |
-| `my-tenant/my-namespace/my-topic` | `persistent://my-tenant/my-namespace/my-topic` |
-
-{{< hint warning >}}
-For non-persistent topics, you need to specify the entire topic name,
-as the default-based rules do not apply for non-partitioned topics.
-Thus, you cannot use a short name like `non-persistent://my-topic` and need to use `non-persistent://public/default/my-topic` instead.
-{{< /hint >}}
-
-#### Subscribing Pulsar Topic Partition
-
-Internally, Pulsar divides a partitioned topic as a set of non-partitioned topics according to the partition size.
-
-For example, if a `simple-string` topic with 3 partitions is created under the `sample` tenant with the `flink` namespace.
-The topics on Pulsar would be:
-
-| Topic name | Partitioned |
-|:------------------------------------------------------|:------------|
-| `persistent://sample/flink/simple-string` | Y |
-| `persistent://sample/flink/simple-string-partition-0` | N |
-| `persistent://sample/flink/simple-string-partition-1` | N |
-| `persistent://sample/flink/simple-string-partition-2` | N |
-
-You can directly consume messages from the topic partitions by using the non-partitioned topic names above.
-For example, use `PulsarSource.builder().setTopics("sample/flink/simple-string-partition-1", "sample/flink/simple-string-partition-2")`
-would consume the partitions 1 and 2 of the `sample/flink/simple-string` topic.
-
-#### Setting Topic Patterns
-
-The Pulsar source extracts the topic type (`persistent` or `non-persistent`) from the provided topic pattern.
-For example, you can use the `PulsarSource.builder().setTopicPattern("non-persistent://my-topic*")` to specify a `non-persistent` topic.
-By default, a `persistent` topic is created if you do not specify the topic type in the regular expression.
-
-You can use `setTopicPattern("topic-*", RegexSubscriptionMode.AllTopics)` to consume
-both `persistent` and `non-persistent` topics based on the topic pattern.
-The Pulsar source would filter the available topics by the `RegexSubscriptionMode`.
-
-### Deserializer
-
-A deserializer (`PulsarDeserializationSchema`) is for decoding Pulsar messages from bytes.
-You can configure the deserializer using `setDeserializationSchema(PulsarDeserializationSchema)`.
-The `PulsarDeserializationSchema` defines how to deserialize a Pulsar `Message<byte[]>`.
-
-If only the raw payload of a message (message data in bytes) is needed,
-you can use the predefined `PulsarDeserializationSchema`. Pulsar connector provides three implementation methods.
-
-- Decode the message by using Pulsar's [Schema](https://pulsar.apache.org/docs/en/schema-understand/).
- ```java
- // Primitive types
- PulsarDeserializationSchema.pulsarSchema(Schema);
-
- // Struct types (JSON, Protobuf, Avro, etc.)
- PulsarDeserializationSchema.pulsarSchema(Schema, Class);
-
- // KeyValue type
- PulsarDeserializationSchema.pulsarSchema(Schema, Class, Class);
- ```
-- Decode the message by using Flink's `DeserializationSchema`
- {{< tabs "pulsar-deserializer-deserialization-schema" >}}
- {{< tab "Java" >}}
- ```java
- PulsarDeserializationSchema.flinkSchema(DeserializationSchema);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- PulsarDeserializationSchema.flink_schema(DeserializationSchema)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Decode the message by using Flink's `TypeInformation`
- {{< tabs "pulsar-deserializer-type-information" >}}
- {{< tab "Java" >}}
- ```java
- PulsarDeserializationSchema.flinkTypeInfo(TypeInformation, ExecutionConfig);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- PulsarDeserializationSchema.flink_type_info(TypeInformation)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-Pulsar `Message<byte[]>` contains some [extra properties](https://pulsar.apache.org/docs/en/concepts-messaging/#messages),
-such as message key, message publish time, message time, and application-defined key/value pairs etc.
-These properties could be defined in the `Message<byte[]>` interface.
-
-If you want to deserialize the Pulsar message by these properties, you need to implement `PulsarDeserializationSchema`.
-Ensure that the `TypeInformation` from the `PulsarDeserializationSchema.getProducedType()` is correct.
-Flink uses this `TypeInformation` to pass the messages to downstream operators.
-
-### Pulsar Subscriptions
-
-A Pulsar subscription is a named configuration rule that determines how messages are delivered to Flink readers.
-The subscription name is required for consuming messages. Pulsar connector supports four subscription types:
-
-- [Exclusive](https://pulsar.apache.org/docs/en/concepts-messaging/#exclusive)
-- [Shared](https://pulsar.apache.org/docs/en/concepts-messaging/#shared)
-- [Failover](https://pulsar.apache.org/docs/en/concepts-messaging/#failover)
-- [Key_Shared](https://pulsar.apache.org/docs/en/concepts-messaging/#key_shared)
-
-There is no difference between `Exclusive` and `Failover` in the Pulsar connector.
-When a Flink reader crashes, all (non-acknowledged and subsequent) messages are redelivered to the available Flink readers.
-
-By default, if no subscription type is defined, Pulsar source uses the `Shared` subscription type.
-
-{{< tabs "pulsar-subscriptions" >}}
-{{< tab "Java" >}}
-
-```java
-// Shared subscription with name "my-shared"
-PulsarSource.builder().setSubscriptionName("my-shared");
-
-// Exclusive subscription with name "my-exclusive"
-PulsarSource.builder().setSubscriptionName("my-exclusive").setSubscriptionType(SubscriptionType.Exclusive);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# Shared subscription with name "my-shared"
-PulsarSource.builder().set_subscription_name("my-shared")
-
-# Exclusive subscription with name "my-exclusive"
-PulsarSource.builder().set_subscription_name("my-exclusive").set_subscription_type(SubscriptionType.Exclusive)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-#### Key_Shared subscriptions
-
-All the Pulsar's messages will be calculated with a key hash in Key_Shared subscription.
-The hash range must be 0 to 65535. We try to compute the key hash in the order of `Message.getOrderingKey()`,
-`Message.getKey()` or `Message.getKeyBytes()`. We will use `"NO_KEY"` str as the message key if none of these keys has been provided.
-
-Pulsar's Key_Shared subscription comes in two forms in Connector, the `KeySharedMode.SPLIT` and `KeySharedMode.JOIN`.
-Different `KeySharedMode` means different split assignment behaviors. If you only consume a subset of Pulsar's key hash range,
-remember to use the `KeySharedMode.JOIN` which will subscribe all the range in only one reader.
-Otherwise, when the ranges can join into a full Pulsar key hash range (0~65535) you should use `KeySharedMode.SPLIT`
-mode for sharing the splits among all the backend readers.
-
-In the `KeySharedMode.SPLIT` mode. The topic will be subscribed by multiple readers.
-But Pulsar has one limit in this situation. That is if a Message can't find the corresponding reader by the key hash range.
-No messages will be delivered to the current readers, until there is a reader which can subscribe to such messages.
-
-##### Define a RangeGenerator
-
-Ensure that you have provided a `RangeGenerator` implementation if you want to use the `Key_Shared` subscription type on the Pulsar connector.
-The `RangeGenerator` generates a set of key hash ranges so that a respective reader subtask only dispatches
-messages where the hash of the message key is contained in the specified range.
-
-The Pulsar connector uses `SplitRangeGenerator` that divides the range by the Flink source
-parallelism if no `RangeGenerator` is provided in the `Key_Shared` subscription type.
-
-Since the Pulsar didn't expose the key hash range method. We have to provide an `FixedKeysRangeGenerator` for end-user.
-You can add the keys you want to consume, no need to calculate any hash ranges.
-The key's hash isn't specified to only one key, so the consuming results may contain the messages with
-different keys comparing the keys you have defined in this range generator.
-Remember to use flink's `DataStream.filter()` method after the Pulsar source.
-
-```java
-FixedKeysRangeGenerator.builder()
- .supportNullKey()
- .key("someKey")
- .keys(Arrays.asList("key1", "key2"))
- .build()
-```
-
-### Starting Position
-
-The Pulsar source is able to consume messages starting from different positions by setting the `setStartCursor(StartCursor)` option.
-Built-in start cursors include:
-
-- Start from the earliest available message in the topic.
- {{< tabs "pulsar-starting-position-earliest" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.earliest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.earliest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Start from the latest available message in the topic.
- {{< tabs "pulsar-starting-position-latest" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.latest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.latest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Start from a specified message between the earliest and the latest.
-The Pulsar connector consumes from the latest available message if the message ID does not exist.
-
- The start message is included in consuming result.
- {{< tabs "pulsar-starting-position-from-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Start from a specified message between the earliest and the latest.
-The Pulsar connector consumes from the latest available message if the message ID doesn't exist.
-
- Include or exclude the start message by using the second boolean parameter.
- {{< tabs "pulsar-starting-position-from-message-id-bool" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageId(MessageId, boolean);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_id(message_id, boolean)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Start from the specified message publish time by `Message<byte[]>.getPublishTime()`.
-This method is deprecated because the name is totally wrong which may cause confuse.
-You can use `StartCursor.fromPublishTime(long)` instead.
-
- {{< tabs "pulsar-starting-position-message-time" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromMessageTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_message_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Start from the specified message publish time by `Message<byte[]>.getPublishTime()`.
- {{< tabs "pulsar-starting-position-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StartCursor.fromPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StartCursor.from_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-{{< hint info >}}
-Each Pulsar message belongs to an ordered sequence on its topic.
-The sequence ID (`MessageId`) of the message is ordered in that sequence.
-The `MessageId` contains some extra information (the ledger, entry, partition) about how the message is stored,
-you can create a `MessageId` by using `DefaultImplementation.newMessageId(long ledgerId, long entryId, int partitionIndex)`.
-{{< /hint >}}
-
-### Boundedness
-
-The Pulsar source supports streaming and batch execution mode.
-By default, the `PulsarSource` is configured for unbounded data.
-
-For unbounded data the Pulsar source never stops until a Flink job is stopped or failed.
-You can use the `setUnboundedStopCursor(StopCursor)` to set the Pulsar source to stop at a specific stop position.
-
-You can use `setBoundedStopCursor(StopCursor)` to specify a stop position for bounded data.
-
-Built-in stop cursors include:
-
-- The Pulsar source never stops consuming messages.
- {{< tabs "pulsar-boundedness-never" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.never();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.never()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop at the latest available message when the Pulsar source starts consuming messages.
- {{< tabs "pulsar-boundedness-latest" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.latest();
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.latest()
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop when the connector meets a given message, or stop at a message which is produced after this given message.
- {{< tabs "pulsar-boundedness-at-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop but include the given message in the consuming result.
- {{< tabs "pulsar-boundedness-after-message-id" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterMessageId(MessageId);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_message_id(message_id)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop at the specified event time by `Message<byte[]>.getEventTime()`. The message with the
-given event time won't be included in the consuming result.
- {{< tabs "pulsar-boundedness-at-event-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atEventTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_event_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop after the specified event time by `Message<byte[]>.getEventTime()`. The message with the
-given event time will be included in the consuming result.
- {{< tabs "pulsar-boundedness-after-event-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterEventTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_event_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop at the specified publish time by `Message<byte[]>.getPublishTime()`. The message with the
-given publish time won't be included in the consuming result.
- {{< tabs "pulsar-boundedness-at-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.atPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.at_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-- Stop after the specified publish time by `Message<byte[]>.getPublishTime()`. The message with the
-given publish time will be included in the consuming result.
- {{< tabs "pulsar-boundedness-after-publish-time" >}}
- {{< tab "Java" >}}
- ```java
- StopCursor.afterPublishTime(long);
- ```
- {{< /tab >}}
- {{< tab "Python" >}}
- ```python
- StopCursor.after_publish_time(int)
- ```
- {{< /tab >}}
- {{< /tabs >}}
-
-### Source Configurable Options
-
-In addition to configuration options described above, you can set arbitrary options for `PulsarClient`,
-`PulsarAdmin`, Pulsar `Consumer` and `PulsarSource` by using `setConfig(ConfigOption<T>, T)`,
-`setConfig(Configuration)` and `setConfig(Properties)`.
-
-#### PulsarClient Options
-
-The Pulsar connector uses the [client API](https://pulsar.apache.org/docs/en/client-libraries-java/)
-to create the `Consumer` instance. The Pulsar connector extracts most parts of Pulsar's `ClientConfigurationData`,
-which is required for creating a `PulsarClient`, as Flink configuration options in `PulsarOptions`.
-
-{{< generated/pulsar_client_configuration >}}
-
-#### PulsarAdmin Options
-
-The [admin API](https://pulsar.apache.org/docs/en/admin-api-overview/) is used for querying topic metadata
-and for discovering the desired topics when the Pulsar connector uses topic-pattern subscription.
-It shares most part of the configuration options with the client API.
-The configuration options listed here are only used in the admin API.
-They are also defined in `PulsarOptions`.
-
-{{< generated/pulsar_admin_configuration >}}
-
-#### Pulsar Consumer Options
-
-In general, Pulsar provides the Reader API and Consumer API for consuming messages in different scenarios.
-The Pulsar connector uses the Consumer API. It extracts most parts of Pulsar's `ConsumerConfigurationData` as Flink configuration options in `PulsarSourceOptions`.
-
-{{< generated/pulsar_consumer_configuration >}}
-
-#### PulsarSource Options
-
-The configuration options below are mainly used for customizing the performance and message acknowledgement behavior.
-You can ignore them if you do not have any performance issues.
-
-{{< generated/pulsar_source_configuration >}}
-
-### Dynamic Partition Discovery
-
-To handle scenarios like topic scaling-out or topic creation without restarting the Flink
-job, the Pulsar source periodically discover new partitions under a provided
-topic-partition subscription pattern. To enable partition discovery, you can set a non-negative value for
-the `PulsarSourceOptions.PULSAR_PARTITION_DISCOVERY_INTERVAL_MS` option:
-
-{{< tabs "pulsar-dynamic-partition-discovery" >}}
-{{< tab "Java" >}}
-
-```java
-// discover new partitions per 10 seconds
-PulsarSource.builder()
- .setConfig(PulsarSourceOptions.PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, 10000);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# discover new partitions per 10 seconds
-PulsarSource.builder()
- .set_config("pulsar.source.partitionDiscoveryIntervalMs", 10000)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-{{< hint warning >}}
-- Partition discovery is **enabled** by default. The Pulsar connector queries the topic metadata every 30 seconds.
-- To disable partition discovery, you need to set a negative partition discovery interval.
-- Partition discovery is disabled for bounded data even if you set this option with a non-negative value.
-{{< /hint >}}
-
-### Event Time and Watermarks
-
-By default, the message uses the timestamp embedded in Pulsar `Message<byte[]>` as the event time.
-You can define your own `WatermarkStrategy` to extract the event time from the message,
-and emit the watermark downstream:
-
-{{< tabs "pulsar-watermarks" >}}
-{{< tab "Java" >}}
-
-```java
-env.fromSource(pulsarSource, new CustomWatermarkStrategy(), "Pulsar Source With Custom Watermark Strategy");
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-env.from_source(pulsar_source, CustomWatermarkStrategy(), "Pulsar Source With Custom Watermark Strategy")
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-[This documentation]({{< ref "docs/dev/datastream/event-time/generating_watermarks.md" >}}) describes
-details about how to define a `WatermarkStrategy`.
-
-### Message Acknowledgement
-
-When a subscription is created, Pulsar [retains](https://pulsar.apache.org/docs/en/concepts-architecture-overview/#persistent-storage) all messages, even if the consumer is disconnected.
-The retained messages are discarded only when the connector acknowledges that all these messages are processed successfully.
-The Pulsar connector supports four subscription types, which makes the acknowledgement behaviors vary among different subscriptions.
-
-#### Acknowledgement on Exclusive and Failover Subscription Types
-
-`Exclusive` and `Failover` subscription types support cumulative acknowledgment. In these subscription types, Flink only needs to acknowledge
-the latest successfully consumed message. All the message before the given message are marked
-with a consumed status.
-
-The Pulsar source acknowledges the current consuming message when checkpoints are **completed**,
-to ensure the consistency between Flink's checkpoint state and committed position on the Pulsar brokers.
-
-If checkpointing is disabled, Pulsar source periodically acknowledges messages.
-You can use the `PulsarSourceOptions.PULSAR_AUTO_COMMIT_CURSOR_INTERVAL` option to set the acknowledgement period.
-
-Pulsar source does **NOT** rely on committed positions for fault tolerance.
-Acknowledging messages is only for exposing the progress of consumers and monitoring on these two subscription types.
-
-#### Acknowledgement on Shared and Key_Shared Subscription Types
-
-In `Shared` and `Key_Shared` subscription types, messages are acknowledged one by one. You can acknowledge
-a message in a transaction and commit it to Pulsar.
-
-You should enable transaction in the Pulsar `borker.conf` file when using these two subscription types in connector:
-
-```text
-transactionCoordinatorEnabled=true
-```
-
-The default timeout for Pulsar transactions is 3 hours.
-Make sure that that timeout is greater than checkpoint interval + maximum recovery time.
-A shorter checkpoint interval indicates a better consuming performance.
-You can use the `PulsarSourceOptions.PULSAR_TRANSACTION_TIMEOUT_MILLIS` option to change the transaction timeout.
-
-If checkpointing is disabled or you can not enable the transaction on Pulsar broker, you should set
-`PulsarSourceOptions.PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE` to `true`.
-The message is immediately acknowledged after consuming.
-No consistency guarantees can be made in this scenario.
-
-{{< hint info >}}
-All acknowledgements in a transaction are recorded in the Pulsar broker side.
-{{< /hint >}}
-
-## Pulsar Sink
-
-The Pulsar Sink supports writing records into one or more Pulsar topics or a specified list of Pulsar partitions.
-
-{{< hint info >}}
-This part describes the Pulsar sink based on the new
-[data sink](https://cwiki.apache.org/confluence/display/FLINK/FLIP-191%3A+Extend+unified+Sink+interface+to+support+small+file+compaction) API.
-
-If you still want to use the legacy `SinkFunction` or on Flink 1.14 or previous releases, just use the StreamNative's
-[pulsar-flink](https://github.com/streamnative/pulsar-flink).
-{{< /hint >}}
-
-### Usage
-
-The Pulsar Sink uses a builder class to construct the `PulsarSink` instance.
-This example writes a String record to a Pulsar topic with at-least-once delivery guarantee.
-
-{{< tabs "46e225b1-1e34-4ff3-890c-aa06a2b99c0a" >}}
-{{< tab "Java" >}}
-
-```java
-DataStream<String> stream = ...
-
-PulsarSink<String> sink = PulsarSink.builder()
- .setServiceUrl(serviceUrl)
- .setAdminUrl(adminUrl)
- .setTopics("topic1")
- .setSerializationSchema(PulsarSerializationSchema.flinkSchema(new SimpleStringSchema()))
- .setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
- .build();
-
-stream.sinkTo(sink);
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-stream = ...
-
-pulsar_sink = PulsarSink.builder() \
- .set_service_url('pulsar://localhost:6650') \
- .set_admin_url('http://localhost:8080') \
- .set_topics("topic1") \
- .set_serialization_schema(PulsarSerializationSchema.flink_schema(SimpleStringSchema())) \
- .set_delivery_guarantee(DeliveryGuarantee.AT_LEAST_ONCE) \
- .build()
-
-stream.sink_to(pulsar_sink)
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-The following properties are **required** for building PulsarSink:
-
-- Pulsar service url, configured by `setServiceUrl(String)`
-- Pulsar service http url (aka. admin url), configured by `setAdminUrl(String)`
-- Topics / partitions to write, see [writing targets](#writing-targets) for more details.
-- Serializer to generate Pulsar messages, see [serializer](#serializer) for more details.
-
-It is recommended to set the producer name in Pulsar Source by `setProducerName(String)`.
-This sets a unique name for the Flink connector in the Pulsar statistic dashboard.
-You can use it to monitor the performance of your Flink connector and applications.
-
-### Producing to topics
-
-Defining the topics for producing is similar to the [topic-partition subscription](#topic-partition-subscription)
-in the Pulsar source. We support a mix-in style of topic setting. You can provide a list of topics,
-partitions, or both of them.
-
-{{< tabs "3d452e6b-bffd-42f7-bb91-974b306262ca" >}}
-{{< tab "Java" >}}
-
-```java
-// Topic "some-topic1" and "some-topic2"
-PulsarSink.builder().setTopics("some-topic1", "some-topic2")
-
-// Partition 0 and 2 of topic "topic-a"
-PulsarSink.builder().setTopics("topic-a-partition-0", "topic-a-partition-2")
-
-// Partition 0 and 2 of topic "topic-a" and topic "some-topic2"
-PulsarSink.builder().setTopics("topic-a-partition-0", "topic-a-partition-2", "some-topic2")
-```
-
-{{< /tab >}}
-{{< tab "Python" >}}
-
-```python
-# Topic "some-topic1" and "some-topic2"
-PulsarSink.builder().set_topics(["some-topic1", "some-topic2"])
-
-# Partition 0 and 2 of topic "topic-a"
-PulsarSink.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2"])
-
-# Partition 0 and 2 of topic "topic-a" and topic "some-topic2"
-PulsarSink.builder().set_topics(["topic-a-partition-0", "topic-a-partition-2", "some-topic2"])
-```
-
-{{< /tab >}}
-{{< /tabs >}}
-
-The topics you provide support auto partition discovery. We query the topic metadata from the Pulsar in a fixed interval.
-You can use the `PulsarSinkOptions.PULSAR_TOPIC_METADATA_REFRESH_INTERVAL` option to change the discovery interval option.
-
-Configuring writing targets can be replaced by using a custom [`TopicRouter`]
-[message routing](#message-routing). Configuring partitions on the Pulsar connector is explained in the [flexible topic naming](#flexible-topic-naming) section.
-
-{{< hint warning >}}
-If you build the Pulsar sink based on both the topic and its corresponding partitions, Pulsar sink merges them and only uses the topic.
-
-For example, when using the `PulsarSink.builder().setTopics("some-topic1", "some-topic1-partition-0")` option to build the Pulsar sink,
-this is simplified to `PulsarSink.builder().setTopics("some-topic1")`.
-{{< /hint >}}
-
-### Serializer
-
-A serializer (`PulsarSerializationSchema`) is required for serializing the record instance into bytes.
-Similar to `PulsarSource`, Pulsar sink supports both Flink's `SerializationSchema` and
-Pulsar's `Schema`. Pulsar's `Schema.AUTO_PRODUCE_BYTES()` is not supported in the Pulsar sink.
-
-If you do not need the message key and other message properties in Pulsar's
-[Message](https://pulsar.apache.org/api/client/2.9.0-SNAPSHOT/org/apache/pulsar/client/api/Message.html) interface,
-you can use the predefined `PulsarSerializationSchema`. The Pulsar sink provides two implementation methods.
-
-- Encode the message by using Pulsar's [Schema](https://pulsar.apache.org/docs/en/schema-understand/).
- ```java
- // Primitive types
- PulsarSerializationSchema.pulsarSchema(Schema)
-
- // Struct types (JSON, Protobuf, Avro, etc.)
- PulsarSerializationSchema.pulsarSchema(Schema, Class)
-
- // KeyValue type
- PulsarSerializationSchema.pulsarSchema(Schema, Class, Class)
- ```
-- Encode the message by using Flink's `SerializationSchema`
-
- {{< tabs "b65b9978-b1d6-4b0d-ade8-78098e0f23d8" >}}
- {{< tab "Java" >}}
-
- ```java
- PulsarSerializationSchema.flinkSchema(SerializationSchema)
- ```
-
- {{< /tab >}}
- {{< tab "Python" >}}
-
- ```python
- PulsarSerializationSchema.flink_schema(SimpleStringSchema())
- ```
-
- {{< /tab >}}
- {{< /tabs >}}
-
-[Schema evolution](https://pulsar.apache.org/docs/en/schema-evolution-compatibility/#schema-evolution)
-can be enabled by users using `PulsarSerializationSchema.pulsarSchema()` and
-`PulsarSinkBuilder.enableSchemaEvolution()`. This means that any broker schema validation is in place.
-
-```java
-Schema<SomePojo> schema = Schema.AVRO(SomePojo.class);
-PulsarSerializationSchema<SomePojo> pulsarSchema = PulsarSerializationSchema.pulsarSchema(schema, SomePojo.class);
-
-PulsarSink<String> sink = PulsarSink.builder()
- ...
- .setSerializationSchema(pulsarSchema)
- .enableSchemaEvolution()
- .build();
-```
-
-{{< hint warning >}}
-If you use Pulsar schema without enabling schema evolution, the target topic will have a `Schema.BYTES` schema.
-Consumers will need to handle the deserialization (if needed) themselves.
-
-For example, if you set `PulsarSerializationSchema.pulsarSchema(Schema.STRING)` without enabling schema evolution,
-the schema stored in Pulsar topics is `Schema.BYTES`.
-{{< /hint >}}
-
-### Message Routing
-
-Routing in Pulsar Sink is operated on the partition level. For a list of partitioned topics,
-the routing algorithm first collects all partitions from different topics, and then calculates routing within all the partitions.
-By default Pulsar Sink supports two router implementation.
-
-- `KeyHashTopicRouter`: use the hashcode of the message's key to decide the topic partition that messages are sent to.
-
- The message key is provided by `PulsarSerializationSchema.key(IN, PulsarSinkContext)`
- You need to implement this interface and extract the message key when you want to send the message with the same key to the same topic partition.
-
- If you do not provide the message key. A topic partition is randomly chosen from the topic list.
-
- The message key can be hashed in two ways: `MessageKeyHash.JAVA_HASH` and `MessageKeyHash.MURMUR3_32_HASH`.
- You can use the `PulsarSinkOptions.PULSAR_MESSAGE_KEY_HASH` option to choose the hash method.
-
-- `RoundRobinRouter`: Round-robin among all the partitions.
-
- All messages are sent to the first partition, and switch to the next partition after sending
- a fixed number of messages. The batch size can be customized by the `PulsarSinkOptions.PULSAR_BATCHING_MAX_MESSAGES` option.
-
-Let’s assume there are ten messages and two topics. Topic A has two partitions while topic B has three partitions.
-The batch size is set to five messages. In this case, topic A has 5 messages per partition which topic B does not receive any messages.
-
-You can configure custom routers by using the `TopicRouter` interface.
-If you implement a `TopicRouter`, ensure that it is serializable.
-And you can return partitions which are not available in the pre-discovered partition list.
-
-Thus, you do not need to specify topics using the `PulsarSinkBuilder.setTopics` option when you implement the custom topic router.
-
-```java
-@PublicEvolving
-public interface TopicRouter<IN> extends Serializable {
-
- String route(IN in, List<String> partitions, PulsarSinkContext context);
-
- default void open(SinkConfiguration sinkConfiguration) {
- // Nothing to do by default.
- }
-}
-```
-
-{{< hint info >}}
-Internally, a Pulsar partition is implemented as a topic. The Pulsar client provides APIs to hide this
-implementation detail and handles routing under the hood automatically. Pulsar Sink uses a lower client
-API to implement its own routing layer to support multiple topics routing.
-
-For details, see [partitioned topics](https://pulsar.apache.org/docs/en/cookbooks-partitioned/).
-{{< /hint >}}
-
-### Delivery Guarantee
-
-`PulsarSink` supports three delivery guarantee semantics.
-
-- `NONE`: Data loss can happen even when the pipeline is running.
- Basically, we use a fire-and-forget strategy to send records to Pulsar topics in this mode.
- It means that this mode has the highest throughput.
-- `AT_LEAST_ONCE`: No data loss happens, but data duplication can happen after a restart from checkpoint.
-- `EXACTLY_ONCE`: No data loss happens. Each record is sent to the Pulsar broker only once.
- Pulsar Sink uses [Pulsar transaction](https://pulsar.apache.org/docs/en/transactions/)
- and two-phase commit (2PC) to ensure records are sent only once even after pipeline restarts.
-
-### Delayed message delivery
-
-[Delayed message delivery](https://pulsar.apache.org/docs/en/next/concepts-messaging/#delayed-message-delivery)
-enables you to delay the possibility to consume a message. With delayed message enabled, the Pulsar sink sends a message to the Pulsar topic
-**immediately**, but the message is delivered to a consumer once the specified delay is over.
-
-Delayed message delivery only works in the `Shared` subscription type. In `Exclusive` and `Failover`
-subscription types, the delayed message is dispatched immediately.
-
-You can configure the `MessageDelayer` to define when to send the message to the consumer.
-The default option is to never delay the message dispatching. You can use the `MessageDelayer.fixed(Duration)` option to
-Configure delaying all messages in a fixed duration. You can also implement the `MessageDelayer`
-interface to dispatch messages at different time.
-
-{{< hint warning >}}
-The dispatch time should be calculated by the `PulsarSinkContext.processTime()`.
-{{< /hint >}}
-
-### Sink Configurable Options
-
-You can set options for `PulsarClient`, `PulsarAdmin`, Pulsar `Producer` and `PulsarSink`
-by using `setConfig(ConfigOption<T>, T)`, `setConfig(Configuration)` and `setConfig(Properties)`.
-
-#### PulsarClient and PulsarAdmin Options
-
-For details, refer to [PulsarAdmin options](#pulsaradmin-options).
-
-#### Pulsar Producer Options
-
-The Pulsar connector uses the Producer API to send messages. It extracts most parts of
-Pulsar's `ProducerConfigurationData` as Flink configuration options in `PulsarSinkOptions`.
-
-{{< generated/pulsar_producer_configuration >}}
-
-#### PulsarSink Options
-
-The configuration options below are mainly used for customizing the performance and message
-sending behavior. You can just leave them alone if you do not have any performance issues.
-
-{{< generated/pulsar_sink_configuration >}}
-
-### Sink Metrics
-
-This table lists supported metrics.
-The first 6 metrics are standard Pulsar Sink metrics as described in
-[FLIP-33: Standardize Connector Metrics]([https://cwiki.apache.org/confluence/display/FLINK/FLIP-33%3A+Standardize+Connector+Metrics](https://cwiki.apache.org/confluence/display/FLINK/FLIP-33%3A+Standardize+Connector+Metrics))
-
-<table class="table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 15%">Scope</th>
- <th class="text-left" style="width: 18%">Metrics</th>
- <th class="text-left" style="width: 18%">User Variables</th>
- <th class="text-left" style="width: 39%">Description</th>
- <th class="text-left" style="width: 10%">Type</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <th rowspan="13">Operator</th>
- <td>numBytesOut</td>
- <td>n/a</td>
- <td>The total number of output bytes since the sink starts. Count towards the numBytesOut in TaskIOMetricsGroup.</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>numBytesOutPerSecond</td>
- <td>n/a</td>
- <td>The output bytes per second</td>
- <td>Meter</td>
- </tr>
- <tr>
- <td>numRecordsOut</td>
- <td>n/a</td>
- <td>The total number of output records since the sink starts.</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>numRecordsOutPerSecond</td>
- <td>n/a</td>
- <td>The output records per second</td>
- <td>Meter</td>
- </tr>
- <tr>
- <td>numRecordsOutErrors</td>
- <td>n/a</td>
- <td>The total number of records failed to send</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>currentSendTime</td>
- <td>n/a</td>
- <td>The time it takes to send the last record, from enqueue the message in client buffer to its ack.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.numAcksReceived</td>
- <td>n/a</td>
- <td>The number of acks received for sent messages.</td>
- <td>Counter</td>
- </tr>
- <tr>
- <td>PulsarSink.sendLatencyMax</td>
- <td>n/a</td>
- <td>The maximum send latency in the last refresh interval across all producers.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency50Pct</td>
- <td>ProducerName</td>
- <td>The 50th percentile of send latency in the last refresh interval for a specific producer.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency75Pct</td>
- <td>ProducerName</td>
- <td>The 75th percentile of send latency in the last refresh interval for a specific producer.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency95Pct</td>
- <td>ProducerName</td>
- <td>The 95th percentile of send latency in the last refresh interval for a specific producer.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency99Pct</td>
- <td>ProducerName</td>
- <td>The 99th percentile of send latency in the last refresh interval for a specific producer.</td>
- <td>Gauge</td>
- </tr>
- <tr>
- <td>PulsarSink.producer."ProducerName".sendLatency999Pct</td>
- <td>ProducerName</td>
- <td>The 99.9th percentile of send latency in the last refresh interval for a specific producer.</td>
- <td>Gauge</td>
- </tr>
- </tbody>
-</table>
-
-{{< hint info >}}
-- `numBytesOut`, `numRecordsOut`, `numRecordsOutErrors` are retrieved from Pulsar client metrics.
-
-- `currentSendTime` tracks the time from when the producer calls `sendAync()` to
- the time when the message is acknowledged by the broker. This metric is not available in `NONE` delivery guarantee.
-{{< /hint >}}
-
-The Pulsar producer refreshes its stats every 60 seconds by default. The PulsarSink retrieves the Pulsar producer
-stats every 500ms. That means that `numRecordsOut`, `numBytesOut`, `numAcksReceived`, and `numRecordsOutErrors`
-are updated every 60 seconds. To increase the metrics refresh frequency, you can change
-the Pulsar producer stats refresh interval to a smaller value (minimum 1 second), as shown below.
-
-{{< tabs "b65b9978-b1d6-4b0d-ade8-78098e0f23d1" >}}
-
-{{< tab "Java" >}}
-```java
-builder.setConfig(PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS, 1L);
-```
-{{< /tab >}}
-
-{{< tab "Python" >}}
-```python
-builder.set_config("pulsar.client.statsIntervalSeconds", "1")
-```
-{{< /tab >}}
-
-{{< /tabs >}}
-
-`numBytesOutRate` and `numRecordsOutRate` are calculated based on the `numBytesOut` and `numRecordsOUt`
-counter respectively. Flink internally uses a fixed 60 seconds window to calculate the rates.
-
-### Brief Design Rationale
-
-Pulsar sink follow the Sink API defined in
-[FLIP-191](https://cwiki.apache.org/confluence/display/FLINK/FLIP-191%3A+Extend+unified+Sink+interface+to+support+small+file+compaction).
-
-#### Stateless SinkWriter
-
-In `EXACTLY_ONCE` mode, the Pulsar sink does not store transaction information in a checkpoint.
-That means that new transactions will be created after a restart.
-Therefore, any message in previous pending transactions is either aborted or timed out
-(They are never visible to the downstream Pulsar consumer).
-The Pulsar team is working to optimize the needed resources by unfinished pending transactions.
-
-#### Pulsar Schema Evolution
-
-[Pulsar Schema Evolution](https://pulsar.apache.org/docs/en/schema-evolution-compatibility/) allows
-you to reuse the same Flink job after certain "allowed" data model changes, like adding or deleting
-a field in a AVRO-based Pojo class. Please note that you can specify Pulsar schema validation rules
-and define an auto schema update. For details, refer to [Pulsar Schema Evolution](https://pulsar.apache.org/docs/en/schema-evolution-compatibility/).
-
-## Upgrading to the Latest Connector Version
-
-The generic upgrade steps are outlined in [upgrading jobs and Flink versions guide]({{< ref "docs/ops/upgrading" >}}).
-The Pulsar connector does not store any state on the Flink side. The Pulsar connector pushes and stores all the states on the Pulsar side.
-For Pulsar, you additionally need to know these limitations:
-
-* Do not upgrade the Pulsar connector and Pulsar broker version at the same time.
-* Always use a newer Pulsar client with Pulsar connector to consume messages from Pulsar.
-
-## Troubleshooting
-
-If you have a problem with Pulsar when using Flink, keep in mind that Flink only wraps
-[PulsarClient](https://pulsar.apache.org/docs/en/client-libraries-java/) or
-[PulsarAdmin](https://pulsar.apache.org/docs/en/admin-api-overview/)
-and your problem might be independent of Flink and sometimes can be solved by upgrading Pulsar brokers,
-reconfiguring Pulsar brokers or reconfiguring Pulsar connector in Flink.
-
-## Known Issues
-
-This section describes some known issues about the Pulsar connectors.
-
-### Unstable on Java 11
-
-Pulsar connector has some known issues on Java 11. It is recommended to run Pulsar connector
-on Java 8.
-
-### No TransactionCoordinatorNotFound, but automatic reconnect
-
-Pulsar transactions are still in active development and are not stable. Pulsar 2.9.2
-introduces [a break change](https://github.com/apache/pulsar/pull/13135) in transactions.
-If you use Pulsar 2.9.2 or higher with an older Pulsar client, you might get a `TransactionCoordinatorNotFound` exception.
-
-You can use the latest `pulsar-client-all` release to resolve this issue.
-
-{{< top >}}
diff --git a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html b/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
deleted file mode 100644
index 31c7273ed53..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
+++ /dev/null
@@ -1,42 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.admin.adminUrl</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The Pulsar service HTTP URL for the admin endpoint. For example, <code class="highlighter-rouge">http://my-broker.example.com:8080</code>, or <code class="highlighter-rouge">https://my-broker.example.com:8443</code> for TLS.</td>
- </tr>
- <tr>
- <td><h5>pulsar.admin.autoCertRefreshTime</h5></td>
- <td style="word-wrap: break-word;">300000</td>
- <td>Integer</td>
- <td>The auto cert refresh time (in ms) if Pulsar admin supports TLS authentication.</td>
- </tr>
- <tr>
- <td><h5>pulsar.admin.connectTimeout</h5></td>
- <td style="word-wrap: break-word;">60000</td>
- <td>Integer</td>
- <td>The connection time out (in ms) for the PulsarAdmin client.</td>
- </tr>
- <tr>
- <td><h5>pulsar.admin.readTimeout</h5></td>
- <td style="word-wrap: break-word;">60000</td>
- <td>Integer</td>
- <td>The server response read timeout (in ms) for the PulsarAdmin client for any request.</td>
- </tr>
- <tr>
- <td><h5>pulsar.admin.requestTimeout</h5></td>
- <td style="word-wrap: break-word;">300000</td>
- <td>Integer</td>
- <td>The server request timeout (in ms) for the PulsarAdmin client for any request.</td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/layouts/shortcodes/generated/pulsar_client_configuration.html b/docs/layouts/shortcodes/generated/pulsar_client_configuration.html
deleted file mode 100644
index 970b847ac2a..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_client_configuration.html
+++ /dev/null
@@ -1,222 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.client.authParamMap</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Map</td>
- <td>Parameters for the authentication plugin.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.authParams</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Parameters for the authentication plugin.<br /><br />Example:<br /><code class="highlighter-rouge">key1:val1,key2:val2</code></td>
- </tr>
- <tr>
- <td><h5>pulsar.client.authPluginClassName</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Name of the authentication plugin.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.concurrentLookupRequest</h5></td>
- <td style="word-wrap: break-word;">5000</td>
- <td>Integer</td>
- <td>The number of concurrent lookup requests allowed to send on each broker connection to prevent overload on the broker. It should be configured with a higher value only in case of it requires to produce or subscribe on thousands of topic using a created <code class="highlighter-rouge">PulsarClient</code></td>
- </tr>
- <tr>
- <td><h5>pulsar.client.connectionTimeoutMs</h5></td>
- <td style="word-wrap: break-word;">10000</td>
- <td>Integer</td>
- <td>Duration (in ms) of waiting for a connection to a broker to be established.<br />If the duration passes without a response from a broker, the connection attempt is dropped.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.connectionsPerBroker</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The maximum number of connections that the client library will open to a single broker.<br /> By default, the connection pool will use a single connection for all the producers and consumers. Increasing this parameter may improve throughput when using many producers over a high latency connection.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.enableBusyWait</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Option to enable busy-wait settings.<br />This option will enable spin-waiting on executors and IO threads in order to reduce latency during context switches. The spinning will consume 100% CPU even when the broker is not doing any work. It is recommended to reduce the number of IO threads and BookKeeper client threads to only have fewer CPU cores busy.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.enableTransaction</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If transaction is enabled, start the <code class="highlighter-rouge">transactionCoordinatorClient</code> with <code class="highlighter-rouge">PulsarClient</code>.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.initialBackoffIntervalNanos</h5></td>
- <td style="word-wrap: break-word;">100000000</td>
- <td>Long</td>
- <td>Default duration (in nanoseconds) for a backoff interval.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.keepAliveIntervalSeconds</h5></td>
- <td style="word-wrap: break-word;">30</td>
- <td>Integer</td>
- <td>Interval (in seconds) for keeping connection between the Pulsar client and broker alive.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.listenerName</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Configure the <code class="highlighter-rouge">listenerName</code> that the broker will return the corresponding <code class="highlighter-rouge">advertisedListener</code>.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.maxBackoffIntervalNanos</h5></td>
- <td style="word-wrap: break-word;">60000000000</td>
- <td>Long</td>
- <td>The maximum duration (in nanoseconds) for a backoff interval.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.maxLookupRedirects</h5></td>
- <td style="word-wrap: break-word;">20</td>
- <td>Integer</td>
- <td>The maximum number of times a lookup-request redirections to a broker.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.maxLookupRequest</h5></td>
- <td style="word-wrap: break-word;">50000</td>
- <td>Integer</td>
- <td>The maximum number of lookup requests allowed on each broker connection to prevent overload on the broker. It should be greater than <code class="highlighter-rouge">pulsar.client.concurrentLookupRequest</code>. Requests that inside <code class="highlighter-rouge">pulsar.client.concurrentLookupRequest</code> are already sent to broker, and requests beyond <code class="highlighter-rouge">pulsar.client.concurrentLookupRequest</code> and under <code class="highlighter-rouge"> [...]
- </tr>
- <tr>
- <td><h5>pulsar.client.maxNumberOfRejectedRequestPerConnection</h5></td>
- <td style="word-wrap: break-word;">50</td>
- <td>Integer</td>
- <td>The maximum number of rejected requests of a broker in a certain period (30s) after the current connection is closed and the client creates a new connection to connect to a different broker.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.memoryLimitBytes</h5></td>
- <td style="word-wrap: break-word;">67108864</td>
- <td>Long</td>
- <td>The limit (in bytes) on the amount of direct memory that will be allocated by this client instance.<br />Note: at this moment this is only limiting the memory for producers. Setting this to <code class="highlighter-rouge">0</code> will disable the limit.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.numIoThreads</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The number of threads used for handling connections to brokers.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.numListenerThreads</h5></td>
- <td style="word-wrap: break-word;">1</td>
- <td>Integer</td>
- <td>The number of threads used for handling message listeners. The listener thread pool is shared across all the consumers and readers that are using a <code class="highlighter-rouge">listener</code> model to get messages. For a given consumer, the listener is always invoked from the same thread to ensure ordering.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.operationTimeoutMs</h5></td>
- <td style="word-wrap: break-word;">30000</td>
- <td>Integer</td>
- <td>Operation timeout (in ms). Operations such as creating producers, subscribing or unsubscribing topics are retried during this interval. If the operation is not completed during this interval, the operation will be marked as failed.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.proxyProtocol</h5></td>
- <td style="word-wrap: break-word;">SNI</td>
- <td><p>Enum</p></td>
- <td>Protocol type to determine the type of proxy routing when a client connects to the proxy using <code class="highlighter-rouge">pulsar.client.proxyServiceUrl</code>.<br /><br />Possible values:<ul><li>"SNI"</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.client.proxyServiceUrl</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Proxy-service URL when a client connects to the broker via the proxy. The client can choose the type of proxy-routing.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.requestTimeoutMs</h5></td>
- <td style="word-wrap: break-word;">60000</td>
- <td>Integer</td>
- <td>Maximum duration (in ms) for completing a request. This config option is not supported before Pulsar 2.8.1</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.serviceUrl</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Service URL provider for Pulsar service.<br />To connect to Pulsar using client libraries, you need to specify a Pulsar protocol URL.<br />You can assign Pulsar protocol URLs to specific clusters and use the Pulsar scheme.<br /><ul><li>This is an example of <code class="highlighter-rouge">localhost</code>: <code class="highlighter-rouge">pulsar://localhost:6650</code>.</li><li>If you have multiple brokers, the URL is as: <code class="highlighter-rouge">pulsar://localhost: [...]
- </tr>
- <tr>
- <td><h5>pulsar.client.sslProvider</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The name of the security provider used for SSL connections. The default value is the default security provider of the JVM.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.statsIntervalSeconds</h5></td>
- <td style="word-wrap: break-word;">60</td>
- <td>Long</td>
- <td>Interval between each stats info.<br /><ul><li>Stats is activated with positive <code class="highlighter-rouge">statsInterval</code></li><li>Set <code class="highlighter-rouge">statsIntervalSeconds</code> to 1 second at least.</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsAllowInsecureConnection</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Whether the Pulsar client accepts untrusted TLS certificate from the broker.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsCiphers</h5></td>
- <td style="word-wrap: break-word;"></td>
- <td>List<String></td>
- <td>A list of cipher suites. This is a named combination of authentication, encryption, MAC and the key exchange algorithm used to negotiate the security settings for a network connection using the TLS or SSL network protocol. By default all the available cipher suites are supported.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsHostnameVerificationEnable</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Whether to enable TLS hostname verification. It allows to validate hostname verification when a client connects to the broker over TLS. It validates incoming x509 certificate and matches provided hostname (CN/SAN) with the expected broker's host name. It follows RFC 2818, 3.1. Server Identity hostname verification.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsProtocols</h5></td>
- <td style="word-wrap: break-word;"></td>
- <td>List<String></td>
- <td>The SSL protocol used to generate the SSLContext. By default, it is set TLS, which is fine for most cases. Allowed values in recent JVMs are TLS, TLSv1.3, TLSv1.2 and TLSv1.1.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsTrustCertsFilePath</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Path to the trusted TLS certificate file.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsTrustStorePassword</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The store password for the key store file.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsTrustStorePath</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The location of the trust store file.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.tlsTrustStoreType</h5></td>
- <td style="word-wrap: break-word;">"JKS"</td>
- <td>String</td>
- <td>The file format of the trust store file.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.useKeyStoreTls</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If TLS is enabled, whether use the KeyStore type as the TLS configuration parameter. If it is set to <code class="highlighter-rouge">false</code>, it means to use the default pem type configuration.</td>
- </tr>
- <tr>
- <td><h5>pulsar.client.useTcpNoDelay</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Whether to use the TCP no-delay flag on the connection to disable Nagle algorithm.<br />No-delay features ensures that packets are sent out on the network as soon as possible, and it is critical to achieve low latency publishes. On the other hand, sending out a huge number of small packets might limit the overall throughput. Therefore, if latency is not a concern, it is recommended to set this option to <code class="highlighter-rouge">false</code>.<br />By default, it is [...]
- </tr>
- </tbody>
-</table>
diff --git a/docs/layouts/shortcodes/generated/pulsar_consumer_configuration.html b/docs/layouts/shortcodes/generated/pulsar_consumer_configuration.html
deleted file mode 100644
index 41f0b24b329..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_consumer_configuration.html
+++ /dev/null
@@ -1,162 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.consumer.ackReceiptEnabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Acknowledgement will return a receipt but this does not mean that the message will not be resent after getting the receipt.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.ackTimeoutMillis</h5></td>
- <td style="word-wrap: break-word;">0</td>
- <td>Long</td>
- <td>The timeout (in ms) for unacknowledged messages, truncated to the nearest millisecond. The timeout needs to be greater than 1 second.<br />By default, the acknowledge timeout is disabled and that means that messages delivered to a consumer will not be re-delivered unless the consumer crashes.<br />When acknowledgement timeout being enabled, if a message is not acknowledged within the specified timeout it will be re-delivered to the consumer (possibly to a different consum [...]
- </tr>
- <tr>
- <td><h5>pulsar.consumer.acknowledgementsGroupTimeMicros</h5></td>
- <td style="word-wrap: break-word;">100000</td>
- <td>Long</td>
- <td>Group a consumer acknowledgment for a specified time (in μs). By default, a consumer uses <code class="highlighter-rouge">100μs</code> grouping time to send out acknowledgments to a broker. If the group time is set to <code class="highlighter-rouge">0</code>, acknowledgments are sent out immediately. A longer ack group time is more efficient at the expense of a slight increase in message re-deliveries after a failure.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.autoAckOldestChunkedMessageOnQueueFull</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Buffering a large number of outstanding uncompleted chunked messages can bring memory pressure and it can be guarded by providing this <code class="highlighter-rouge">pulsar.consumer.maxPendingChunkedMessage</code> threshold. Once a consumer reaches this threshold, it drops the outstanding unchunked-messages by silently acknowledging if <code class="highlighter-rouge">pulsar.consumer.autoAckOldestChunkedMessageOnQueueFull</code> is true. Otherwise, it marks them for redel [...]
- </tr>
- <tr>
- <td><h5>pulsar.consumer.consumerName</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>The consumer name is informative and it can be used to identify a particular consumer instance from the topic stats.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.cryptoFailureAction</h5></td>
- <td style="word-wrap: break-word;">FAIL</td>
- <td><p>Enum</p></td>
- <td>The consumer should take action when it receives a message that can not be decrypted.<ul><li><code class="highlighter-rouge">FAIL</code>: this is the default option to fail messages until crypto succeeds.</li><li><code class="highlighter-rouge">DISCARD</code>: silently acknowledge but do not deliver messages to an application.</li><li><code class="highlighter-rouge">CONSUME</code>: deliver encrypted messages to applications. It is the application's responsibility to decry [...]
- </tr>
- <tr>
- <td><h5>pulsar.consumer.deadLetterPolicy.deadLetterTopic</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Name of the dead topic where the failed messages are sent.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.deadLetterPolicy.maxRedeliverCount</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Integer</td>
- <td>The maximum number of times that a message are redelivered before being sent to the dead letter queue.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.deadLetterPolicy.retryLetterTopic</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Name of the retry topic where the failed messages are sent.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.expireTimeOfIncompleteChunkedMessageMillis</h5></td>
- <td style="word-wrap: break-word;">60000</td>
- <td>Long</td>
- <td>If a producer fails to publish all the chunks of a message, the consumer can expire incomplete chunks if the consumer cannot receive all chunks in expire times (default 1 hour, in ms).</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.maxPendingChunkedMessage</h5></td>
- <td style="word-wrap: break-word;">10</td>
- <td>Integer</td>
- <td>The consumer buffers chunk messages into memory until it receives all the chunks of the original message. While consuming chunk-messages, chunks from the same message might not be contiguous in the stream and they might be mixed with other messages' chunks. So, consumer has to maintain multiple buffers to manage chunks coming from different messages. This mainly happens when multiple publishers are publishing messages on the topic concurrently or publishers failed to publ [...]
- </tr>
- <tr>
- <td><h5>pulsar.consumer.maxTotalReceiverQueueSizeAcrossPartitions</h5></td>
- <td style="word-wrap: break-word;">50000</td>
- <td>Integer</td>
- <td>The maximum total receiver queue size across partitions.<br />This setting reduces the receiver queue size for individual partitions if the total receiver queue size exceeds this value.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.negativeAckRedeliveryDelayMicros</h5></td>
- <td style="word-wrap: break-word;">60000000</td>
- <td>Long</td>
- <td>Delay (in μs) to wait before redelivering messages that failed to be processed.<br />When an application uses <code class="highlighter-rouge">Consumer.negativeAcknowledge(Message)</code>, failed messages are redelivered after a fixed timeout.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.poolMessages</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Enable pooling of messages and the underlying data buffers.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.priorityLevel</h5></td>
- <td style="word-wrap: break-word;">0</td>
- <td>Integer</td>
- <td>Priority level for a consumer to which a broker gives more priorities while dispatching messages in the shared subscription type.<br />The broker follows descending priorities. For example, 0=max-priority, 1, 2,...<br />In shared subscription mode, the broker first dispatches messages to the consumers on the highest priority level if they have permits. Otherwise, the broker considers consumers on the next priority level.<br /><br />Example 1<br />If a subscription has con [...]
-C1, 0, 2
-C2, 0, 1
-C3, 0, 1
-C4, 1, 2
-C5, 1, 1
-<br />The order in which a broker dispatches messages to consumers is: C1, C2, C3, C1, C4, C5, C4.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.properties</h5></td>
- <td style="word-wrap: break-word;"></td>
- <td>Map</td>
- <td>A name or value property of this consumer. <code class="highlighter-rouge">properties</code> is application defined metadata attached to a consumer. When getting a topic stats, associate this metadata with the consumer stats for easier identification.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.readCompacted</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If enabling <code class="highlighter-rouge">readCompacted</code>, a consumer reads messages from a compacted topic rather than reading a full message backlog of a topic.<br />A consumer only sees the latest value for each key in the compacted topic, up until reaching the point in the topic message when compacting backlog. Beyond that point, send messages as normal.<br />Only enabling <code class="highlighter-rouge">readCompacted</code> on subscriptions to persistent topic [...]
- </tr>
- <tr>
- <td><h5>pulsar.consumer.receiverQueueSize</h5></td>
- <td style="word-wrap: break-word;">1000</td>
- <td>Integer</td>
- <td>Size of a consumer's receiver queue.<br />For example, the number of messages accumulated by a consumer before an application calls <code class="highlighter-rouge">Receive</code>.<br />A value higher than the default value increases consumer throughput, though at the expense of more memory utilization.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.replicateSubscriptionState</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If <code class="highlighter-rouge">replicateSubscriptionState</code> is enabled, a subscription state is replicated to geo-replicated clusters.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.retryEnable</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If enabled, the consumer will automatically retry messages.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.subscriptionMode</h5></td>
- <td style="word-wrap: break-word;">Durable</td>
- <td><p>Enum</p></td>
- <td>Select the subscription mode to be used when subscribing to the topic.<ul><li><code class="highlighter-rouge">Durable</code>: Make the subscription to be backed by a durable cursor that will retain messages and persist the current position.</li><li><code class="highlighter-rouge">NonDurable</code>: Lightweight subscription mode that doesn't have a durable cursor associated</li></ul><br /><br />Possible values:<ul><li>"Durable"</li><li>"NonDurable"</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.subscriptionName</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Specify the subscription name for this consumer. This argument is required when constructing the consumer.</td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.subscriptionType</h5></td>
- <td style="word-wrap: break-word;">Shared</td>
- <td><p>Enum</p></td>
- <td>Subscription type.<br /><br />Four subscription types are available:<ul><li>Exclusive</li><li>Failover</li><li>Shared</li><li>Key_Shared</li></ul><br /><br />Possible values:<ul><li>"Exclusive"</li><li>"Shared"</li><li>"Failover"</li><li>"Key_Shared"</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.consumer.tickDurationMillis</h5></td>
- <td style="word-wrap: break-word;">1000</td>
- <td>Long</td>
- <td>Granularity (in ms) of the ack-timeout redelivery.<br />A greater (for example, 1 hour) <code class="highlighter-rouge">tickDurationMillis</code> reduces the memory overhead to track messages.</td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/layouts/shortcodes/generated/pulsar_producer_configuration.html b/docs/layouts/shortcodes/generated/pulsar_producer_configuration.html
deleted file mode 100644
index 798490306e1..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_producer_configuration.html
+++ /dev/null
@@ -1,78 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.producer.batchingEnabled</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>Enable batch send ability, it was enabled by default.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.batchingMaxBytes</h5></td>
- <td style="word-wrap: break-word;">131072</td>
- <td>Integer</td>
- <td>The maximum size of messages permitted in a batch. Keep the maximum consistent as previous versions.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.batchingMaxMessages</h5></td>
- <td style="word-wrap: break-word;">1000</td>
- <td>Integer</td>
- <td>The maximum number of messages permitted in a batch.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.batchingMaxPublishDelayMicros</h5></td>
- <td style="word-wrap: break-word;">1000</td>
- <td>Long</td>
- <td>Batching time period of sending messages.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.batchingPartitionSwitchFrequencyByPublishDelay</h5></td>
- <td style="word-wrap: break-word;">10</td>
- <td>Integer</td>
- <td>The maximum wait time for switching topic partitions.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.chunkingEnabled</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td></td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.compressionType</h5></td>
- <td style="word-wrap: break-word;">NONE</td>
- <td><p>Enum</p></td>
- <td>Message data compression type used by a producer.Available options:<ul><li><a href="LZ4">https://github.com/lz4/lz4</a></li><li><a href="ZLIB">https://zlib.net/</a></li><li><a href="ZSTD">https://facebook.github.io/zstd/</a></li><li><a href="SNAPPY">https://google.github.io/snappy/</a></li></ul><br /><br />Possible values:<ul><li>"NONE"</li><li>"LZ4"</li><li>"ZLIB"</li><li>"ZSTD"</li><li>"SNAPPY"</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.initialSequenceId</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>Long</td>
- <td>The sequence id for avoiding the duplication, it's used when Pulsar doesn't have transaction.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.producerName</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>A producer name which would be displayed in the Pulsar's dashboard. If no producer name was provided, we would use a Pulsar generated name instead.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.properties</h5></td>
- <td style="word-wrap: break-word;"></td>
- <td>Map</td>
- <td>A name or value property of this consumer. <code class="highlighter-rouge">properties</code> is application defined metadata attached to a consumer. When getting a topic stats, associate this metadata with the consumer stats for easier identification.</td>
- </tr>
- <tr>
- <td><h5>pulsar.producer.sendTimeoutMs</h5></td>
- <td style="word-wrap: break-word;">30000</td>
- <td>Long</td>
- <td>Message send timeout in ms.If a message is not acknowledged by a server before the <code class="highlighter-rouge">sendTimeout</code> expires, an error occurs.</td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/layouts/shortcodes/generated/pulsar_sink_configuration.html b/docs/layouts/shortcodes/generated/pulsar_sink_configuration.html
deleted file mode 100644
index f207e5fd325..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_sink_configuration.html
+++ /dev/null
@@ -1,54 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.sink.deliveryGuarantee</h5></td>
- <td style="word-wrap: break-word;">none</td>
- <td><p>Enum</p></td>
- <td>Optional delivery guarantee when committing.<br /><br />Possible values:<ul><li>"exactly-once": Records are only delivered exactly-once also under failover scenarios. To build a complete exactly-once pipeline is required that the source and sink support exactly-once and are properly configured.</li><li>"at-least-once": Records are ensured to be delivered but it may happen that the same record is delivered multiple times. Usually, this guarantee is faster than the exactly- [...]
- </tr>
- <tr>
- <td><h5>pulsar.sink.enableMetrics</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>The metrics from Pulsar Producer are only exposed if you enable this option.You should set the <code class="highlighter-rouge">pulsar.client.statsIntervalSeconds</code> to a positive value if you enable this option.</td>
- </tr>
- <tr>
- <td><h5>pulsar.sink.enableSchemaEvolution</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If you enable this option and use PulsarSerializationSchema.pulsarSchema(), we would consume and deserialize the message by using Pulsar's <code class="highlighter-rouge">Schema</code>.</td>
- </tr>
- <tr>
- <td><h5>pulsar.sink.maxRecommitTimes</h5></td>
- <td style="word-wrap: break-word;">5</td>
- <td>Integer</td>
- <td>The allowed transaction recommit times if we meet some retryable exception. This is used in Pulsar Transaction.</td>
- </tr>
- <tr>
- <td><h5>pulsar.sink.messageKeyHash</h5></td>
- <td style="word-wrap: break-word;">murmur-3-32-hash</td>
- <td><p>Enum</p></td>
- <td>The hash policy for routing message by calculating the hash code of message key.<br /><br />Possible values:<ul><li>"java-hash": This hash would use <code class="highlighter-rouge">String.hashCode()</code> to calculate the message key string's hash code.</li><li>"murmur-3-32-hash": This hash would calculate message key's hash code by using <a href="https://en.wikipedia.org/wiki/MurmurHash">Murmur3</a> algorithm.</li></ul></td>
- </tr>
- <tr>
- <td><h5>pulsar.sink.topicMetadataRefreshInterval</h5></td>
- <td style="word-wrap: break-word;">1800000</td>
- <td>Long</td>
- <td>Auto update the topic metadata in a fixed interval (in ms). The default value is 30 minutes.</td>
- </tr>
- <tr>
- <td><h5>pulsar.sink.transactionTimeoutMillis</h5></td>
- <td style="word-wrap: break-word;">10800000</td>
- <td>Long</td>
- <td>This option is used when the user require the <code class="highlighter-rouge">DeliveryGuarantee.EXACTLY_ONCE</code> semantic.We would use transaction for making sure the message could be write only once.</td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/layouts/shortcodes/generated/pulsar_source_configuration.html b/docs/layouts/shortcodes/generated/pulsar_source_configuration.html
deleted file mode 100644
index e28ffd5746a..00000000000
--- a/docs/layouts/shortcodes/generated/pulsar_source_configuration.html
+++ /dev/null
@@ -1,66 +0,0 @@
-<table class="configuration table table-bordered">
- <thead>
- <tr>
- <th class="text-left" style="width: 20%">Key</th>
- <th class="text-left" style="width: 15%">Default</th>
- <th class="text-left" style="width: 10%">Type</th>
- <th class="text-left" style="width: 55%">Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><h5>pulsar.source.allowKeySharedOutOfOrderDelivery</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>If enabled, it will relax the ordering requirement, allowing the broker to send out-of-order messages in case of failures. This will make it faster for new consumers to join without being stalled by an existing slow consumer.<br />In this case, a single consumer will still receive all the keys, but they may be coming in different orders.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.autoCommitCursorInterval</h5></td>
- <td style="word-wrap: break-word;">5000</td>
- <td>Long</td>
- <td>This option is used only when the user disables the checkpoint and uses Exclusive or Failover subscription. We would automatically commit the cursor using the given period (in ms).</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.enableAutoAcknowledgeMessage</h5></td>
- <td style="word-wrap: break-word;">false</td>
- <td>Boolean</td>
- <td>Flink commits the consuming position with pulsar transactions on checkpoint. However, if you have disabled the Flink checkpoint or disabled transaction for your Pulsar cluster, ensure that you have set this option to <code class="highlighter-rouge">true</code>.<br />The source would use pulsar client's internal mechanism and commit cursor in two ways.<ul><li>For <code class="highlighter-rouge">Key_Shared</code> and <code class="highlighter-rouge">Shared</code> subscriptio [...]
- </tr>
- <tr>
- <td><h5>pulsar.source.enableMetrics</h5></td>
- <td style="word-wrap: break-word;">true</td>
- <td>Boolean</td>
- <td>The metrics from Pulsar Consumer are only exposed if you enable this option.You should set the <code class="highlighter-rouge">pulsar.client.statsIntervalSeconds</code> to a positive value if you enable this option.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.maxFetchRecords</h5></td>
- <td style="word-wrap: break-word;">100</td>
- <td>Integer</td>
- <td>The maximum number of records to fetch to wait when polling. A longer time increases throughput but also latency. A fetch batch might be finished earlier because of <code class="highlighter-rouge">pulsar.source.maxFetchTime</code>.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.maxFetchTime</h5></td>
- <td style="word-wrap: break-word;">10000</td>
- <td>Long</td>
- <td>The maximum time (in ms) to wait when fetching records. A longer time increases throughput but also latency. A fetch batch might be finished earlier because of <code class="highlighter-rouge">pulsar.source.maxFetchRecords</code>.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.partitionDiscoveryIntervalMs</h5></td>
- <td style="word-wrap: break-word;">30000</td>
- <td>Long</td>
- <td>The interval (in ms) for the Pulsar source to discover the new partitions. A non-positive value disables the partition discovery.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.transactionTimeoutMillis</h5></td>
- <td style="word-wrap: break-word;">10800000</td>
- <td>Long</td>
- <td>This option is used in <code class="highlighter-rouge">Shared</code> or <code class="highlighter-rouge">Key_Shared</code> subscription. You should configure this option when you do not enable the <code class="highlighter-rouge">pulsar.source.enableAutoAcknowledgeMessage</code> option.<br />The value (in ms) should be greater than the checkpoint interval.</td>
- </tr>
- <tr>
- <td><h5>pulsar.source.verifyInitialOffsets</h5></td>
- <td style="word-wrap: break-word;">WARN_ON_MISMATCH</td>
- <td><p>Enum</p></td>
- <td>Upon (re)starting the source, check whether the expected message can be read. If failure is enabled, the application fails. Otherwise, it logs a warning. A possible solution is to adjust the retention settings in Pulsar or ignoring the check result.<br /><br />Possible values:<ul><li>"FAIL_ON_MISMATCH": Fail the consuming from Pulsar when we don't find the related cursor.</li><li>"WARN_ON_MISMATCH": Print a warn message and start consuming from the valid offset.</li></ul></td>
- </tr>
- </tbody>
-</table>
diff --git a/docs/setup_docs.sh b/docs/setup_docs.sh
index f66f5a44e70..8025ac03fa1 100755
--- a/docs/setup_docs.sh
+++ b/docs/setup_docs.sh
@@ -54,6 +54,7 @@ cd tmp
integrate_connector_docs elasticsearch v3.0.0
integrate_connector_docs aws v3.0.0
integrate_connector_docs cassandra v3.0.0
+integrate_connector_docs pulsar main
cd ..
rm -rf tmp
diff --git a/flink-architecture-tests/flink-architecture-tests-production/pom.xml b/flink-architecture-tests/flink-architecture-tests-production/pom.xml
index e203c020022..26c761151ae 100644
--- a/flink-architecture-tests/flink-architecture-tests-production/pom.xml
+++ b/flink-architecture-tests/flink-architecture-tests-production/pom.xml
@@ -184,11 +184,6 @@ under the License.
<artifactId>flink-connector-kinesis</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-pulsar</artifactId>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-rabbitmq</artifactId>
diff --git a/flink-architecture-tests/pom.xml b/flink-architecture-tests/pom.xml
index c79875d4f78..ec3e7be9830 100644
--- a/flink-architecture-tests/pom.xml
+++ b/flink-architecture-tests/pom.xml
@@ -216,13 +216,6 @@ under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-pulsar</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-rabbitmq</artifactId>
diff --git a/flink-connectors/flink-connector-pulsar/archunit-violations/3ac3a1dc-681f-4213-9990-b7b3298a20bc b/flink-connectors/flink-connector-pulsar/archunit-violations/3ac3a1dc-681f-4213-9990-b7b3298a20bc
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/flink-connectors/flink-connector-pulsar/archunit-violations/f4d91193-72ba-4ce4-ad83-98f780dce581 b/flink-connectors/flink-connector-pulsar/archunit-violations/f4d91193-72ba-4ce4-ad83-98f780dce581
deleted file mode 100644
index 40e7dc9686c..00000000000
--- a/flink-connectors/flink-connector-pulsar/archunit-violations/f4d91193-72ba-4ce4-ad83-98f780dce581
+++ /dev/null
@@ -1,18 +0,0 @@
-org.apache.flink.connector.pulsar.sink.PulsarSinkITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.pulsar.source.PulsarSourceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.pulsar.source.PulsarUnorderedSourceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-pulsar/archunit-violations/stored.rules b/flink-connectors/flink-connector-pulsar/archunit-violations/stored.rules
deleted file mode 100644
index efb12f34f7b..00000000000
--- a/flink-connectors/flink-connector-pulsar/archunit-violations/stored.rules
+++ /dev/null
@@ -1,4 +0,0 @@
-#
-#Thu Mar 03 12:42:13 CST 2022
-Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=3ac3a1dc-681f-4213-9990-b7b3298a20bc
-ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=f4d91193-72ba-4ce4-ad83-98f780dce581
diff --git a/flink-connectors/flink-connector-pulsar/pom.xml b/flink-connectors/flink-connector-pulsar/pom.xml
deleted file mode 100644
index f84db57ed89..00000000000
--- a/flink-connectors/flink-connector-pulsar/pom.xml
+++ /dev/null
@@ -1,326 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>flink-connectors</artifactId>
- <groupId>org.apache.flink</groupId>
- <version>1.17-SNAPSHOT</version>
- </parent>
-
- <artifactId>flink-connector-pulsar</artifactId>
- <name>Flink : Connectors : Pulsar</name>
-
- <packaging>jar</packaging>
-
- <properties>
- <pulsar.version>2.10.2</pulsar.version>
-
- <!-- Test Libraries -->
- <protobuf-maven-plugin.version>0.6.1</protobuf-maven-plugin.version>
- <os-maven-plugin.version>1.7.0</os-maven-plugin.version>
- <pulsar-netty.version>4.1.77.Final</pulsar-netty.version>
- <pulsar-grpc.version>1.45.1</pulsar-grpc.version>
- <pulsar-caffeine.version>2.9.1</pulsar-caffeine.version>
- </properties>
-
- <dependencies>
-
- <!-- Core -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!-- Connectors -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <!-- Protobuf & Protobuf Native Schema support. Add it to your pom if you need protobuf -->
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${protoc.version}</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
- <!-- Tests -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <!-- Pulsar testing environment -->
-
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>pulsar</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- Pulsar bundles the latest bookkeeper -->
- <!-- we don't override the version here. -->
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>testmocks</artifactId>
- <version>${pulsar.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-testng</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-broker</artifactId>
- <version>${pulsar.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- <version>${pulsar-caffeine.version}</version>
- <scope>test</scope>
- </dependency>
-
- <!-- Add Pulsar 2.x as a dependency. -->
- <!-- Move this to button for avoiding class conflicts with pulsar-broker. -->
-
- <dependency>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-client-all</artifactId>
- <version>${pulsar.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.activation</groupId>
- <artifactId>javax.activation</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jakarta.activation</groupId>
- <artifactId>jakarta.activation-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jakarta.ws.rs</groupId>
- <artifactId>jakarta.ws.rs-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jakarta.xml.bind</groupId>
- <artifactId>jakarta.xml.bind-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.jcip</groupId>
- <artifactId>jcip-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.pulsar</groupId>
- <artifactId>pulsar-package-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- ArchUit test dependencies -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-architecture-tests-test</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
-
- <dependencyManagement>
- <dependencies>
- <!-- Pulsar use higher gRPC version. -->
- <dependency>
- <groupId>io.grpc</groupId>
- <artifactId>grpc-bom</artifactId>
- <version>${pulsar-grpc.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
-
- <!-- Pulsar use higher netty version. -->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-bom</artifactId>
- <version>${pulsar-netty.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <build>
- <extensions>
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
- <version>${os-maven-plugin.version}</version>
- </extension>
- </extensions>
-
- <plugins>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <version>${protobuf-maven-plugin.version}</version>
- <extensions>true</extensions>
- <configuration>
- <!-- Currently Flink azure test pipeline would first pre-compile and then upload the compiled
- directory, then it download the directory and run the corresponding tests. However, the protoc
- executable under the target directory would lost the execution permission bit after downloading.
- To solve this issue we would skip generating the target files if they already exist after
- downloading. Meanwhile, since the time might be not consistent between the pre-compile and
- the actual execution, we need to adjust the timestamp manually, see unpack_build_artifact.sh-->
- <checkStaleness>true</checkStaleness>
- <protoTestSourceRoot>${project.basedir}/src/test/resources/protobuf
- </protoTestSourceRoot>
- <!-- Generates classes into a separate directory since the generator always removes existing files. -->
- <outputDirectory>
- ${project.build.directory}/generated-test-sources/protobuf/java
- </outputDirectory>
- <protocArtifact>
- com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
- </protocArtifact>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>test-compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <!-- Adding protobuf generated classes to test build path -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-test-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>
- ${project.build.directory}/generated-test-sources/protobuf/java
- </source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <configuration>
- <includes>
- <include>**/testutils/**</include>
- <include>META-INF/LICENSE</include>
- <include>META-INF/NOTICE</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-test-sources</id>
- <goals>
- <goal>test-jar-no-fork</goal>
- </goals>
- <configuration>
- <archive>
- <!-- Globally exclude maven metadata, because it may accidentally bundle files we don't intend to -->
- <addMavenDescriptor>false</addMavenDescriptor>
- </archive>
- <includes>
- <include>**/testutils/**</include>
- <include>META-INF/LICENSE</include>
- <include>META-INF/NOTICE</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
deleted file mode 100644
index 1f01b242143..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.config;
-
-import org.apache.flink.annotation.Internal;
-
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminBuilder;
-import org.apache.pulsar.client.api.Authentication;
-import org.apache.pulsar.client.api.AuthenticationFactory;
-import org.apache.pulsar.client.api.ClientBuilder;
-import org.apache.pulsar.client.api.ProxyProtocol;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.impl.auth.AuthenticationDisabled;
-
-import java.util.Map;
-import java.util.TreeSet;
-
-import static java.util.Collections.singletonMap;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAMS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAM_MAP;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PLUGIN_CLASS_NAME;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTO_CERT_REFRESH_TIME;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_CONCURRENT_LOOKUP_REQUEST;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_CONNECTIONS_PER_BROKER;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_CONNECTION_TIMEOUT_MS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_CONNECT_TIMEOUT;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_BUSY_WAIT;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_TRANSACTION;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_INITIAL_BACKOFF_INTERVAL_NANOS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_KEEP_ALIVE_INTERVAL_SECONDS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_LISTENER_NAME;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MAX_BACKOFF_INTERVAL_NANOS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MAX_LOOKUP_REDIRECTS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MAX_LOOKUP_REQUEST;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MAX_NUMBER_OF_REJECTED_REQUEST_PER_CONNECTION;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MEMORY_LIMIT_BYTES;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_NUM_IO_THREADS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_NUM_LISTENER_THREADS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_OPERATION_TIMEOUT_MS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_PROXY_PROTOCOL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_PROXY_SERVICE_URL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_READ_TIMEOUT;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_REQUEST_TIMEOUT;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_REQUEST_TIMEOUT_MS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SSL_PROVIDER;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_ALLOW_INSECURE_CONNECTION;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_CIPHERS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_HOSTNAME_VERIFICATION_ENABLE;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_PROTOCOLS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_TRUST_CERTS_FILE_PATH;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_TRUST_STORE_PASSWORD;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_TRUST_STORE_PATH;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_TLS_TRUST_STORE_TYPE;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_USE_KEY_STORE_TLS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_USE_TCP_NO_DELAY;
-import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient;
-import static org.apache.pulsar.client.api.SizeUnit.BYTES;
-
-/** The factory for creating pulsar client classes from {@link PulsarConfiguration}. */
-@Internal
-public final class PulsarClientFactory {
-
- private PulsarClientFactory() {
- // No need to create instance.
- }
-
- /** Create a PulsarClient by using the flink Configuration and the config customizer. */
- public static PulsarClient createClient(PulsarConfiguration configuration) {
- ClientBuilder builder = PulsarClient.builder();
-
- // requestTimeoutMs don't have a setter method on ClientBuilder. We have to use low level
- // setter method instead. So we put this at the beginning of the builder.
- Integer requestTimeoutMs = configuration.get(PULSAR_REQUEST_TIMEOUT_MS);
- builder.loadConf(singletonMap("requestTimeoutMs", requestTimeoutMs));
-
- // Create the authentication instance for the Pulsar client.
- builder.authentication(createAuthentication(configuration));
-
- configuration.useOption(PULSAR_SERVICE_URL, builder::serviceUrl);
- configuration.useOption(PULSAR_LISTENER_NAME, builder::listenerName);
- configuration.useOption(
- PULSAR_OPERATION_TIMEOUT_MS,
- timeout -> builder.operationTimeout(timeout, MILLISECONDS));
- configuration.useOption(PULSAR_NUM_IO_THREADS, builder::ioThreads);
- configuration.useOption(PULSAR_NUM_LISTENER_THREADS, builder::listenerThreads);
- configuration.useOption(PULSAR_CONNECTIONS_PER_BROKER, builder::connectionsPerBroker);
- configuration.useOption(PULSAR_USE_TCP_NO_DELAY, builder::enableTcpNoDelay);
- configuration.useOption(PULSAR_TLS_TRUST_CERTS_FILE_PATH, builder::tlsTrustCertsFilePath);
- configuration.useOption(
- PULSAR_TLS_ALLOW_INSECURE_CONNECTION, builder::allowTlsInsecureConnection);
- configuration.useOption(
- PULSAR_TLS_HOSTNAME_VERIFICATION_ENABLE, builder::enableTlsHostnameVerification);
- configuration.useOption(PULSAR_USE_KEY_STORE_TLS, builder::useKeyStoreTls);
- configuration.useOption(PULSAR_SSL_PROVIDER, builder::sslProvider);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_TYPE, builder::tlsTrustStoreType);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_PATH, builder::tlsTrustStorePath);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_PASSWORD, builder::tlsTrustStorePassword);
- configuration.useOption(PULSAR_TLS_CIPHERS, TreeSet::new, builder::tlsCiphers);
- configuration.useOption(PULSAR_TLS_PROTOCOLS, TreeSet::new, builder::tlsProtocols);
- configuration.useOption(
- PULSAR_MEMORY_LIMIT_BYTES, bytes -> builder.memoryLimit(bytes, BYTES));
- configuration.useOption(
- PULSAR_STATS_INTERVAL_SECONDS, v -> builder.statsInterval(v, SECONDS));
- configuration.useOption(
- PULSAR_CONCURRENT_LOOKUP_REQUEST, builder::maxConcurrentLookupRequests);
- configuration.useOption(PULSAR_MAX_LOOKUP_REQUEST, builder::maxLookupRequests);
- configuration.useOption(PULSAR_MAX_LOOKUP_REDIRECTS, builder::maxLookupRedirects);
- configuration.useOption(
- PULSAR_MAX_NUMBER_OF_REJECTED_REQUEST_PER_CONNECTION,
- builder::maxNumberOfRejectedRequestPerConnection);
- configuration.useOption(
- PULSAR_KEEP_ALIVE_INTERVAL_SECONDS, v -> builder.keepAliveInterval(v, SECONDS));
- configuration.useOption(
- PULSAR_CONNECTION_TIMEOUT_MS, v -> builder.connectionTimeout(v, MILLISECONDS));
- configuration.useOption(
- PULSAR_INITIAL_BACKOFF_INTERVAL_NANOS,
- v -> builder.startingBackoffInterval(v, NANOSECONDS));
- configuration.useOption(
- PULSAR_MAX_BACKOFF_INTERVAL_NANOS, v -> builder.maxBackoffInterval(v, NANOSECONDS));
- configuration.useOption(PULSAR_ENABLE_BUSY_WAIT, builder::enableBusyWait);
- if (configuration.contains(PULSAR_PROXY_SERVICE_URL)) {
- String proxyServiceUrl = configuration.get(PULSAR_PROXY_SERVICE_URL);
- ProxyProtocol proxyProtocol = configuration.get(PULSAR_PROXY_PROTOCOL);
- builder.proxyServiceUrl(proxyServiceUrl, proxyProtocol);
- }
- configuration.useOption(PULSAR_ENABLE_TRANSACTION, builder::enableTransaction);
-
- return sneakyClient(builder::build);
- }
-
- /**
- * PulsarAdmin shares almost the same configuration with PulsarClient, but we separate this
- * creating method for directly use it.
- */
- public static PulsarAdmin createAdmin(PulsarConfiguration configuration) {
- PulsarAdminBuilder builder = PulsarAdmin.builder();
-
- // Create the authentication instance for the Pulsar client.
- builder.authentication(createAuthentication(configuration));
-
- configuration.useOption(PULSAR_ADMIN_URL, builder::serviceHttpUrl);
- configuration.useOption(PULSAR_TLS_TRUST_CERTS_FILE_PATH, builder::tlsTrustCertsFilePath);
- configuration.useOption(
- PULSAR_TLS_ALLOW_INSECURE_CONNECTION, builder::allowTlsInsecureConnection);
- configuration.useOption(
- PULSAR_TLS_HOSTNAME_VERIFICATION_ENABLE, builder::enableTlsHostnameVerification);
- configuration.useOption(PULSAR_USE_KEY_STORE_TLS, builder::useKeyStoreTls);
- configuration.useOption(PULSAR_SSL_PROVIDER, builder::sslProvider);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_TYPE, builder::tlsTrustStoreType);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_PATH, builder::tlsTrustStorePath);
- configuration.useOption(PULSAR_TLS_TRUST_STORE_PASSWORD, builder::tlsTrustStorePassword);
- configuration.useOption(PULSAR_TLS_CIPHERS, TreeSet::new, builder::tlsCiphers);
- configuration.useOption(PULSAR_TLS_PROTOCOLS, TreeSet::new, builder::tlsProtocols);
- configuration.useOption(
- PULSAR_CONNECT_TIMEOUT, v -> builder.connectionTimeout(v, MILLISECONDS));
- configuration.useOption(PULSAR_READ_TIMEOUT, v -> builder.readTimeout(v, MILLISECONDS));
- configuration.useOption(
- PULSAR_REQUEST_TIMEOUT, v -> builder.requestTimeout(v, MILLISECONDS));
- configuration.useOption(
- PULSAR_AUTO_CERT_REFRESH_TIME, v -> builder.autoCertRefreshTime(v, MILLISECONDS));
-
- return sneakyClient(builder::build);
- }
-
- /**
- * Create the {@link Authentication} instance for both {@code PulsarClient} and {@code
- * PulsarAdmin}. If the user didn't provide configuration, a {@link AuthenticationDisabled}
- * instance would be returned.
- *
- * <p>This method behavior is the same as the pulsar command line tools.
- */
- private static Authentication createAuthentication(PulsarConfiguration configuration) {
- if (configuration.contains(PULSAR_AUTH_PLUGIN_CLASS_NAME)) {
- String authPluginClassName = configuration.get(PULSAR_AUTH_PLUGIN_CLASS_NAME);
-
- if (configuration.contains(PULSAR_AUTH_PARAMS)) {
- String authParamsString = configuration.get(PULSAR_AUTH_PARAMS);
- return sneakyClient(
- () -> AuthenticationFactory.create(authPluginClassName, authParamsString));
- } else {
- Map<String, String> paramsMap = configuration.getProperties(PULSAR_AUTH_PARAM_MAP);
- if (paramsMap.isEmpty()) {
- throw new IllegalArgumentException(
- String.format(
- "No %s or %s provided",
- PULSAR_AUTH_PARAMS.key(), PULSAR_AUTH_PARAM_MAP.key()));
- }
-
- return sneakyClient(
- () -> AuthenticationFactory.create(authPluginClassName, paramsMap));
- }
- }
-
- return AuthenticationDisabled.INSTANCE;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigBuilder.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigBuilder.java
deleted file mode 100644
index b8ce86f7e72..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigBuilder.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.config;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.Configuration;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.function.Function;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * A builder for building the unmodifiable {@link Configuration} instance. Providing the common
- * validate logic for Pulsar source & sink.
- */
-@Internal
-public final class PulsarConfigBuilder {
-
- private final Configuration configuration = new Configuration();
-
- /** Validate if the config has a existed option. */
- public <T> boolean contains(ConfigOption<T> option) {
- return configuration.contains(option);
- }
-
- /**
- * Get an option-related config value. We would return the default config value defined in the
- * option if no value existed instead.
- *
- * @param key Config option instance.
- */
- public <T> T get(ConfigOption<T> key) {
- return configuration.get(key);
- }
-
- /**
- * Add a config option with a not null value. The config key shouldn't be duplicated.
- *
- * @param option Config option instance, contains key & type definition.
- * @param value The config value which shouldn't be null.
- */
- public <T> void set(ConfigOption<T> option, T value) {
- checkNotNull(option);
- checkNotNull(value);
-
- if (configuration.contains(option)) {
- T oldValue = configuration.get(option);
- checkArgument(
- Objects.equals(oldValue, value),
- "This option %s has been set to value %s.",
- option.key(),
- oldValue);
- } else {
- configuration.set(option, value);
- }
- }
-
- /**
- * Fill in a set of configs which shouldn't be duplicated.
- *
- * @param config A set of configs.
- */
- public void set(Configuration config) {
- Map<String, String> existedConfigs = configuration.toMap();
- List<String> duplicatedKeys = new ArrayList<>();
- for (Map.Entry<String, String> entry : config.toMap().entrySet()) {
- String key = entry.getKey();
- if (existedConfigs.containsKey(key)) {
- String value2 = existedConfigs.get(key);
- if (!Objects.equals(value2, entry.getValue())) {
- duplicatedKeys.add(key);
- }
- }
- }
- checkArgument(
- duplicatedKeys.isEmpty(),
- "Invalid configuration, these keys %s are already exist with different config value.",
- duplicatedKeys);
- configuration.addAll(config);
- }
-
- /**
- * Fill in a set of config properties which shouldn't be duplicated.
- *
- * @param properties A config which could be string type.
- */
- public void set(Properties properties) {
- properties.keySet().stream()
- .map(String::valueOf)
- .forEach(
- key -> {
- ConfigOption<String> option =
- ConfigOptions.key(key).stringType().noDefaultValue();
- Object value = properties.get(key);
-
- if (value != null) {
- set(option, value.toString());
- }
- });
- }
-
- /**
- * Override the option with the given value. It will not check the existed option comparing to
- * {@link #set(ConfigOption, Object)}.
- */
- public <T> void override(ConfigOption<T> option, T value) {
- checkNotNull(option);
- checkNotNull(value);
-
- configuration.set(option, value);
- }
-
- /** Validate the current config instance and return a unmodifiable configuration. */
- public <T extends PulsarConfiguration> T build(
- PulsarConfigValidator validator, Function<Configuration, T> constructor) {
- validator.validate(configuration);
- return constructor.apply(configuration);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigValidator.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigValidator.java
deleted file mode 100644
index ac8f2968de4..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfigValidator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.config;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.Configuration;
-
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableSet;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-
-/**
- * A config validator for building {@link PulsarConfiguration} in {@link PulsarConfigBuilder}. It's
- * used for source & sink builder.
- *
- * <p>We would validate:
- *
- * <ul>
- * <li>If the user has provided the required config options.
- * <li>If the user has provided some conflict options.
- * </ul>
- */
-@Internal
-public class PulsarConfigValidator {
-
- private final List<Set<ConfigOption<?>>> conflictOptions;
- private final Set<ConfigOption<?>> requiredOptions;
-
- private PulsarConfigValidator(
- List<Set<ConfigOption<?>>> conflictOptions, Set<ConfigOption<?>> requiredOptions) {
- this.conflictOptions = conflictOptions;
- this.requiredOptions = requiredOptions;
- }
-
- /** Package private validating for using in {@link PulsarConfigBuilder}. */
- void validate(Configuration configuration) {
- requiredOptions.forEach(
- option ->
- checkArgument(
- configuration.contains(option),
- "Config option %s is not provided for pulsar client.",
- option));
- conflictOptions.forEach(
- options -> {
- long nums = options.stream().filter(configuration::contains).count();
- checkArgument(
- nums <= 1,
- "Conflict config options %s were provided, we only support one of them for creating pulsar client.",
- options);
- });
- }
-
- /** Return the builder for building {@link PulsarConfigValidator}. */
- public static PulsarConfigValidatorBuilder builder() {
- return new PulsarConfigValidatorBuilder();
- }
-
- /** Builder pattern for building {@link PulsarConfigValidator}. */
- public static class PulsarConfigValidatorBuilder {
-
- private final List<Set<ConfigOption<?>>> conflictOptions = new ArrayList<>();
- private final Set<ConfigOption<?>> requiredOptions = new HashSet<>();
-
- public PulsarConfigValidatorBuilder conflictOptions(ConfigOption<?>... options) {
- checkArgument(options.length > 1, "You should provide at least two conflict options.");
- conflictOptions.add(ImmutableSet.copyOf(options));
- return this;
- }
-
- public PulsarConfigValidatorBuilder requiredOption(ConfigOption<?> option) {
- requiredOptions.add(option);
- return this;
- }
-
- public PulsarConfigValidator build() {
- ImmutableList<Set<ConfigOption<?>>> conflict = ImmutableList.copyOf(conflictOptions);
- Set<ConfigOption<?>> required = ImmutableSet.copyOf(requiredOptions);
-
- return new PulsarConfigValidator(conflict, required);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfiguration.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfiguration.java
deleted file mode 100644
index 0681a3ef4cb..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarConfiguration.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.config;
-
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.UnmodifiableConfiguration;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
-import static java.util.function.Function.identity;
-import static java.util.stream.Collectors.toList;
-
-/**
- * An unmodifiable {@link Configuration} for Pulsar. We provide extra methods for building the
- * different Pulsar client instance.
- */
-public abstract class PulsarConfiguration extends UnmodifiableConfiguration {
- private static final long serialVersionUID = 3050894147145572345L;
-
- /**
- * Creates a new PulsarConfiguration, which holds a copy of the given configuration that can't
- * be altered.
- *
- * @param config The configuration with the original contents.
- */
- protected PulsarConfiguration(Configuration config) {
- super(config);
- }
-
- /**
- * Get the option value by a prefix. We would return an empty map if the option doesn't exist.
- */
- public Map<String, String> getProperties(ConfigOption<Map<String, String>> option) {
- Map<String, String> properties = new HashMap<>();
- if (contains(option)) {
- Map<String, String> map = get(option);
- properties.putAll(map);
- }
-
- // Filter the sub config option. These options could be provided by SQL.
- String prefix = option.key() + ".";
- List<String> keys =
- keySet().stream()
- .filter(key -> key.startsWith(prefix) && key.length() > prefix.length())
- .collect(toList());
-
- // Put these config options' value into return result.
- for (String key : keys) {
- ConfigOption<String> o = ConfigOptions.key(key).stringType().noDefaultValue();
- String value = get(o);
- properties.put(key.substring(prefix.length()), value);
- }
-
- return properties;
- }
-
- /** Get an option value from the given config, convert it into a new value instance. */
- public <F, T> T get(ConfigOption<F> option, Function<F, T> convertor) {
- F value = get(option);
- if (value != null) {
- return convertor.apply(value);
- } else {
- return null;
- }
- }
-
- /** Set the config option's value to a given builder. */
- public <T> void useOption(ConfigOption<T> option, Consumer<T> setter) {
- useOption(option, identity(), setter);
- }
-
- /**
- * Query the config option's value, convert it into a required type, set it to a given builder.
- */
- public <T, V> void useOption(
- ConfigOption<T> option, Function<T, V> convertor, Consumer<V> setter) {
- if (contains(option) || option.hasDefaultValue()) {
- V value = get(option, convertor);
- setter.accept(value);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
deleted file mode 100644
index f878cb4efe0..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.config;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.docs.ConfigGroup;
-import org.apache.flink.annotation.docs.ConfigGroups;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.description.Description;
-
-import org.apache.pulsar.client.api.ProxyProtocol;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.flink.configuration.description.TextElement.code;
-import static org.apache.flink.configuration.description.TextElement.text;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.ADMIN_CONFIG_PREFIX;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.CLIENT_CONFIG_PREFIX;
-
-/**
- * Configuration for Pulsar Client, these config options would be used for both source, sink and
- * table.
- */
-@PublicEvolving
-@ConfigGroups(
- groups = {
- @ConfigGroup(name = "PulsarClient", keyPrefix = CLIENT_CONFIG_PREFIX),
- @ConfigGroup(name = "PulsarAdmin", keyPrefix = ADMIN_CONFIG_PREFIX)
- })
-public final class PulsarOptions {
-
- // Pulsar client API config prefix.
- public static final String CLIENT_CONFIG_PREFIX = "pulsar.client.";
- // Pulsar admin API config prefix.
- public static final String ADMIN_CONFIG_PREFIX = "pulsar.admin.";
-
- private PulsarOptions() {
- // This is a constant class
- }
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // The configuration for ClientConfigurationData part.
- // All the configuration listed below should have the pulsar.client prefix.
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- public static final ConfigOption<String> PULSAR_SERVICE_URL =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "serviceUrl")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text("Service URL provider for Pulsar service.")
- .linebreak()
- .text(
- "To connect to Pulsar using client libraries, you need to specify a Pulsar protocol URL.")
- .linebreak()
- .text(
- "You can assign Pulsar protocol URLs to specific clusters and use the Pulsar scheme.")
- .linebreak()
- .list(
- text(
- "This is an example of %s: %s.",
- code("localhost"),
- code("pulsar://localhost:6650")),
- text(
- "If you have multiple brokers, the URL is as: %s",
- code(
- "pulsar://localhost:6550,localhost:6651,localhost:6652")),
- text(
- "A URL for a production Pulsar cluster is as: %s",
- code(
- "pulsar://pulsar.us-west.example.com:6650")),
- text(
- "If you use TLS authentication, the URL is as %s",
- code(
- "pulsar+ssl://pulsar.us-west.example.com:6651")))
- .build());
-
- public static final ConfigOption<String> PULSAR_AUTH_PLUGIN_CLASS_NAME =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "authPluginClassName")
- .stringType()
- .noDefaultValue()
- .withDescription("Name of the authentication plugin.");
-
- public static final ConfigOption<String> PULSAR_AUTH_PARAMS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "authParams")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text("Parameters for the authentication plugin.")
- .linebreak()
- .linebreak()
- .text("Example:")
- .linebreak()
- .add(code("key1:val1,key2:val2"))
- .build());
-
- public static final ConfigOption<Map<String, String>> PULSAR_AUTH_PARAM_MAP =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "authParamMap")
- .mapType()
- .noDefaultValue()
- .withDescription("Parameters for the authentication plugin.");
-
- public static final ConfigOption<Integer> PULSAR_OPERATION_TIMEOUT_MS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "operationTimeoutMs")
- .intType()
- .defaultValue(30000)
- .withDescription(
- Description.builder()
- .text("Operation timeout (in ms).")
- .text(
- " Operations such as creating producers, subscribing or unsubscribing topics are retried during this interval.")
- .text(
- " If the operation is not completed during this interval, the operation will be marked as failed.")
- .build());
-
- public static final ConfigOption<Long> PULSAR_STATS_INTERVAL_SECONDS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "statsIntervalSeconds")
- .longType()
- .defaultValue(60L)
- .withDescription(
- Description.builder()
- .text("Interval between each stats info.")
- .linebreak()
- .list(
- text(
- "Stats is activated with positive %s",
- code("statsInterval")),
- text(
- "Set %s to 1 second at least.",
- code("statsIntervalSeconds")))
- .build());
-
- public static final ConfigOption<Integer> PULSAR_NUM_IO_THREADS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "numIoThreads")
- .intType()
- .defaultValue(1)
- .withDescription(
- "The number of threads used for handling connections to brokers.");
-
- public static final ConfigOption<Integer> PULSAR_NUM_LISTENER_THREADS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "numListenerThreads")
- .intType()
- .defaultValue(1)
- .withDescription(
- Description.builder()
- .text(
- "The number of threads used for handling message listeners.")
- .text(
- " The listener thread pool is shared across all the consumers and readers that are using a %s model to get messages.",
- code("listener"))
- .text(
- " For a given consumer, the listener is always invoked from the same thread to ensure ordering.")
- .build());
-
- public static final ConfigOption<Integer> PULSAR_CONNECTIONS_PER_BROKER =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "connectionsPerBroker")
- .intType()
- .defaultValue(1)
- .withDescription(
- Description.builder()
- .text(
- "The maximum number of connections that the client library will open to a single broker.")
- .linebreak()
- .text(
- " By default, the connection pool will use a single connection for all the producers and consumers.")
- .text(
- " Increasing this parameter may improve throughput when using many producers over a high latency connection.")
- .build());
-
- public static final ConfigOption<Boolean> PULSAR_USE_TCP_NO_DELAY =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "useTcpNoDelay")
- .booleanType()
- .defaultValue(true)
- .withDescription(
- Description.builder()
- .text(
- "Whether to use the TCP no-delay flag on the connection to disable Nagle algorithm.")
- .linebreak()
- .text(
- "No-delay features ensures that packets are sent out on the network as soon as possible,")
- .text(" and it is critical to achieve low latency publishes.")
- .text(
- " On the other hand, sending out a huge number of small packets might limit the overall throughput.")
- .text(
- " Therefore, if latency is not a concern, it is recommended to set this option to %s.",
- code("false"))
- .linebreak()
- .text("By default, it is set to %s.", code("true"))
- .build());
-
- public static final ConfigOption<String> PULSAR_TLS_TRUST_CERTS_FILE_PATH =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsTrustCertsFilePath")
- .stringType()
- .defaultValue("")
- .withDescription("Path to the trusted TLS certificate file.");
-
- public static final ConfigOption<Boolean> PULSAR_TLS_ALLOW_INSECURE_CONNECTION =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsAllowInsecureConnection")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- "Whether the Pulsar client accepts untrusted TLS certificate from the broker.");
-
- public static final ConfigOption<Boolean> PULSAR_TLS_HOSTNAME_VERIFICATION_ENABLE =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsHostnameVerificationEnable")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- Description.builder()
- .text("Whether to enable TLS hostname verification.")
- .text(
- " It allows to validate hostname verification when a client connects to the broker over TLS.")
- .text(
- " It validates incoming x509 certificate and matches provided hostname (CN/SAN) with the expected broker's host name.")
- .text(
- " It follows RFC 2818, 3.1. Server Identity hostname verification.")
- .build());
-
- public static final ConfigOption<Integer> PULSAR_CONCURRENT_LOOKUP_REQUEST =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "concurrentLookupRequest")
- .intType()
- .defaultValue(5000)
- .withDescription(
- Description.builder()
- .text(
- "The number of concurrent lookup requests allowed to send on each broker connection to prevent overload on the broker.")
- .text(
- " It should be configured with a higher value only in case of it requires to produce or subscribe on thousands of topic using a created %s",
- code("PulsarClient"))
- .build());
-
- public static final ConfigOption<Integer> PULSAR_MAX_LOOKUP_REQUEST =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "maxLookupRequest")
- .intType()
- .defaultValue(50000)
- .withDescription(
- Description.builder()
- .text(
- "The maximum number of lookup requests allowed on each broker connection to prevent overload on the broker.")
- .text(
- " It should be greater than %s.",
- code("pulsar.client.concurrentLookupRequest"))
- .text(
- " Requests that inside %s are already sent to broker,",
- code("pulsar.client.concurrentLookupRequest"))
- .text(
- " and requests beyond %s and under %s will wait in each client cnx.",
- code("pulsar.client.concurrentLookupRequest"),
- code("maxLookupRequests"))
- .build());
-
- public static final ConfigOption<Integer> PULSAR_MAX_LOOKUP_REDIRECTS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "maxLookupRedirects")
- .intType()
- .defaultValue(20)
- .withDescription(
- "The maximum number of times a lookup-request redirections to a broker.");
-
- public static final ConfigOption<Integer> PULSAR_MAX_NUMBER_OF_REJECTED_REQUEST_PER_CONNECTION =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "maxNumberOfRejectedRequestPerConnection")
- .intType()
- .defaultValue(50)
- .withDescription(
- Description.builder()
- .text(
- "The maximum number of rejected requests of a broker in a certain period (30s) after the current connection is closed")
- .text(
- " and the client creates a new connection to connect to a different broker.")
- .build());
-
- public static final ConfigOption<Integer> PULSAR_KEEP_ALIVE_INTERVAL_SECONDS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "keepAliveIntervalSeconds")
- .intType()
- .defaultValue(30)
- .withDescription(
- "Interval (in seconds) for keeping connection between the Pulsar client and broker alive.");
-
- public static final ConfigOption<Integer> PULSAR_CONNECTION_TIMEOUT_MS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "connectionTimeoutMs")
- .intType()
- .defaultValue(10000)
- .withDescription(
- Description.builder()
- .text(
- "Duration (in ms) of waiting for a connection to a broker to be established.")
- .linebreak()
- .text(
- "If the duration passes without a response from a broker, the connection attempt is dropped.")
- .build());
-
- public static final ConfigOption<Integer> PULSAR_REQUEST_TIMEOUT_MS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "requestTimeoutMs")
- .intType()
- .defaultValue(60000)
- .withDescription(
- "Maximum duration (in ms) for completing a request. This config option is not supported before Pulsar 2.8.1");
-
- public static final ConfigOption<Long> PULSAR_INITIAL_BACKOFF_INTERVAL_NANOS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "initialBackoffIntervalNanos")
- .longType()
- .defaultValue(TimeUnit.MILLISECONDS.toNanos(100))
- .withDescription("Default duration (in nanoseconds) for a backoff interval.");
-
- public static final ConfigOption<Long> PULSAR_MAX_BACKOFF_INTERVAL_NANOS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "maxBackoffIntervalNanos")
- .longType()
- .defaultValue(SECONDS.toNanos(60))
- .withDescription(
- "The maximum duration (in nanoseconds) for a backoff interval.");
-
- public static final ConfigOption<Boolean> PULSAR_ENABLE_BUSY_WAIT =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "enableBusyWait")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- Description.builder()
- .text("Option to enable busy-wait settings.")
- .linebreak()
- .text(
- "This option will enable spin-waiting on executors and IO threads in order to reduce latency during context switches.")
- .text(
- " The spinning will consume 100% CPU even when the broker is not doing any work.")
- .text(
- " It is recommended to reduce the number of IO threads and BookKeeper client threads to only have fewer CPU cores busy.")
- .build());
-
- public static final ConfigOption<String> PULSAR_LISTENER_NAME =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "listenerName")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text(
- "Configure the %s that the broker will return the corresponding %s.",
- code("listenerName"), code("advertisedListener"))
- .build());
-
- public static final ConfigOption<Boolean> PULSAR_USE_KEY_STORE_TLS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "useKeyStoreTls")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- Description.builder()
- .text(
- "If TLS is enabled, whether use the KeyStore type as the TLS configuration parameter.")
- .text(
- " If it is set to %s, it means to use the default pem type configuration.",
- code("false"))
- .build());
-
- public static final ConfigOption<String> PULSAR_SSL_PROVIDER =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "sslProvider")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text(
- "The name of the security provider used for SSL connections.")
- .text(
- " The default value is the default security provider of the JVM.")
- .build());
-
- public static final ConfigOption<String> PULSAR_TLS_TRUST_STORE_TYPE =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsTrustStoreType")
- .stringType()
- .defaultValue("JKS")
- .withDescription("The file format of the trust store file.");
-
- public static final ConfigOption<String> PULSAR_TLS_TRUST_STORE_PATH =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsTrustStorePath")
- .stringType()
- .noDefaultValue()
- .withDescription("The location of the trust store file.");
-
- public static final ConfigOption<String> PULSAR_TLS_TRUST_STORE_PASSWORD =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsTrustStorePassword")
- .stringType()
- .noDefaultValue()
- .withDescription("The store password for the key store file.");
-
- // The real config type is Set<String>, you should provided a json str here.
- public static final ConfigOption<List<String>> PULSAR_TLS_CIPHERS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsCiphers")
- .stringType()
- .asList()
- .defaultValues()
- .withDescription(
- Description.builder()
- .text("A list of cipher suites.")
- .text(
- " This is a named combination of authentication, encryption,")
- .text(
- " MAC and the key exchange algorithm used to negotiate the security settings for a network connection using the TLS or SSL network protocol.")
- .text(
- " By default all the available cipher suites are supported.")
- .build());
-
- public static final ConfigOption<List<String>> PULSAR_TLS_PROTOCOLS =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tlsProtocols")
- .stringType()
- .asList()
- .defaultValues()
- .withDescription(
- Description.builder()
- .text("The SSL protocol used to generate the SSLContext.")
- .text(
- " By default, it is set TLS, which is fine for most cases.")
- .text(
- " Allowed values in recent JVMs are TLS, TLSv1.3, TLSv1.2 and TLSv1.1.")
- .build());
-
- public static final ConfigOption<Long> PULSAR_MEMORY_LIMIT_BYTES =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "memoryLimitBytes")
- .longType()
- .defaultValue(64 * 1024 * 1024L)
- .withDescription(
- Description.builder()
- .text(
- "The limit (in bytes) on the amount of direct memory that will be allocated by this client instance.")
- .linebreak()
- .text(
- "Note: at this moment this is only limiting the memory for producers.")
- .text(" Setting this to %s will disable the limit.", code("0"))
- .build());
-
- public static final ConfigOption<String> PULSAR_PROXY_SERVICE_URL =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "proxyServiceUrl")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text(
- "Proxy-service URL when a client connects to the broker via the proxy.")
- .text(" The client can choose the type of proxy-routing.")
- .build());
-
- public static final ConfigOption<ProxyProtocol> PULSAR_PROXY_PROTOCOL =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "proxyProtocol")
- .enumType(ProxyProtocol.class)
- .defaultValue(ProxyProtocol.SNI)
- .withDescription(
- Description.builder()
- .text(
- "Protocol type to determine the type of proxy routing when a client connects to the proxy using %s.",
- code("pulsar.client.proxyServiceUrl"))
- .build());
-
- public static final ConfigOption<Boolean> PULSAR_ENABLE_TRANSACTION =
- ConfigOptions.key(CLIENT_CONFIG_PREFIX + "enableTransaction")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- Description.builder()
- .text(
- "If transaction is enabled, start the %s with %s.",
- code("transactionCoordinatorClient"),
- code("PulsarClient"))
- .build());
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // The configuration for PulsarAdmin part.
- // All the configuration listed below should have the pulsar.admin prefix.
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- public static final ConfigOption<String> PULSAR_ADMIN_URL =
- ConfigOptions.key(ADMIN_CONFIG_PREFIX + "adminUrl")
- .stringType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text(
- "The Pulsar service HTTP URL for the admin endpoint. For example, %s, or %s for TLS.",
- code("http://my-broker.example.com:8080"),
- code("https://my-broker.example.com:8443"))
- .build());
-
- public static final ConfigOption<Integer> PULSAR_CONNECT_TIMEOUT =
- ConfigOptions.key(ADMIN_CONFIG_PREFIX + "connectTimeout")
- .intType()
- .defaultValue(60000)
- .withDescription("The connection time out (in ms) for the PulsarAdmin client.");
-
- public static final ConfigOption<Integer> PULSAR_READ_TIMEOUT =
- ConfigOptions.key(ADMIN_CONFIG_PREFIX + "readTimeout")
- .intType()
- .defaultValue(60000)
- .withDescription(
- "The server response read timeout (in ms) for the PulsarAdmin client for any request.");
-
- public static final ConfigOption<Integer> PULSAR_REQUEST_TIMEOUT =
- ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestTimeout")
- .intType()
- .defaultValue(300000)
- .withDescription(
- "The server request timeout (in ms) for the PulsarAdmin client for any request.");
-
- public static final ConfigOption<Integer> PULSAR_AUTO_CERT_REFRESH_TIME =
- ConfigOptions.key(ADMIN_CONFIG_PREFIX + "autoCertRefreshTime")
- .intType()
- .defaultValue(300000)
- .withDescription(
- "The auto cert refresh time (in ms) if Pulsar admin supports TLS authentication.");
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/MetricNames.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/MetricNames.java
deleted file mode 100644
index 5156b472098..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/MetricNames.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.metrics;
-
-/** The constant class for holding all the custom metrics names in Pulsar. */
-public final class MetricNames {
-
- private MetricNames() {
- // No public constructor.
- }
-
- public static final String PULSAR_PRODUCER_METRIC_NAME = "PulsarProducer";
- public static final String NUM_MSGS_SENT = "numMsgsSent";
- public static final String NUM_BYTES_SENT = "numBytesSent";
- public static final String NUM_SEND_FAILED = "numSendFailed";
- public static final String NUM_ACKS_RECEIVED = "numAcksReceived";
- public static final String SEND_MSGS_RATE = "sendMsgsRate";
- public static final String SEND_BYTES_RATE = "sendBytesRate";
- public static final String SEND_LATENCY_MILLIS_50_PCT = "sendLatencyMillis50pct";
- public static final String SEND_LATENCY_MILLIS_75_PCT = "sendLatencyMillis75pct";
- public static final String SEND_LATENCY_MILLIS_95_PCT = "sendLatencyMillis95pct";
- public static final String SEND_LATENCY_MILLIS_99_PCT = "sendLatencyMillis99pct";
- public static final String SEND_LATENCY_MILLIS_999_PCT = "sendLatencyMillis999pct";
- public static final String SEND_LATENCY_MILLIS_MAX = "sendLatencyMillisMax";
- public static final String TOTAL_MSGS_SENT = "totalMsgsSent";
- public static final String TOTAL_BYTES_SENT = "totalBytesSent";
- public static final String TOTAL_SEND_FAILED = "totalSendFailed";
- public static final String TOTAL_ACKS_RECEIVED = "totalAcksReceived";
- public static final String PENDING_QUEUE_SIZE = "pendingQueueSize";
-
- public static final String PULSAR_CONSUMER_METRIC_NAME = "PulsarConsumer";
- public static final String NUM_MSGS_RECEIVED = "numMsgsReceived";
- public static final String NUM_BYTES_RECEIVED = "numBytesReceived";
- public static final String RATE_MSGS_RECEIVED = "rateMsgsReceived";
- public static final String RATE_BYTES_RECEIVED = "rateBytesReceived";
- public static final String NUM_ACKS_SENT = "numAcksSent";
- public static final String NUM_ACKS_FAILED = "numAcksFailed";
- public static final String NUM_RECEIVE_FAILED = "numReceiveFailed";
- public static final String NUM_BATCH_RECEIVE_FAILED = "numBatchReceiveFailed";
- public static final String TOTAL_MSGS_RECEIVED = "totalMsgsReceived";
- public static final String TOTAL_BYTES_RECEIVED = "totalBytesReceived";
- public static final String TOTAL_RECEIVED_FAILED = "totalReceivedFailed";
- public static final String TOTAL_BATCH_RECEIVED_FAILED = "totalBatchReceivedFailed";
- public static final String TOTAL_ACKS_SENT = "totalAcksSent";
- public static final String TOTAL_ACKS_FAILED = "totalAcksFailed";
- public static final String MSG_NUM_IN_RECEIVER_QUEUE = "msgNumInReceiverQueue";
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/ProducerMetricsInterceptor.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/ProducerMetricsInterceptor.java
deleted file mode 100644
index e2b99152042..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/metrics/ProducerMetricsInterceptor.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.metrics;
-
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
-
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.interceptor.ProducerInterceptor;
-
-/** The metric statistic for Pulsar's {@link Producer}. */
-public class ProducerMetricsInterceptor implements ProducerInterceptor {
-
- private final Counter numRecordsOutErrors;
- private final Counter numRecordsOutCounter;
- private final Counter numBytesOutCounter;
-
- public ProducerMetricsInterceptor(SinkWriterMetricGroup metricGroup) {
- this.numRecordsOutErrors = metricGroup.getNumRecordsOutErrorsCounter();
- this.numRecordsOutCounter = metricGroup.getIOMetricGroup().getNumRecordsOutCounter();
- this.numBytesOutCounter = metricGroup.getIOMetricGroup().getNumBytesOutCounter();
- }
-
- @Override
- public void close() {
- // Nothing to do by default.
- }
-
- @Override
- public boolean eligible(Message message) {
- return true;
- }
-
- @Override
- public Message beforeSend(Producer producer, Message message) {
- return message;
- }
-
- @Override
- public void onSendAcknowledgement(
- Producer producer, Message message, MessageId msgId, Throwable exception) {
- if (exception != null) {
- numRecordsOutErrors.inc(1);
- } else {
- numRecordsOutCounter.inc(1);
- numBytesOutCounter.inc(message.size());
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchema.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchema.java
deleted file mode 100644
index fe5dacac02b..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchema.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema;
-
-import org.apache.flink.annotation.Internal;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
-import org.apache.pulsar.common.schema.KeyValue;
-import org.apache.pulsar.common.schema.KeyValueEncodingType;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.createSchema;
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.encodeClassInfo;
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.haveProtobuf;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkState;
-import static org.apache.flink.util.ReflectionUtil.getTemplateType1;
-import static org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo.decodeKeyValueEncodingType;
-import static org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo.decodeKeyValueSchemaInfo;
-import static org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo.encodeKeyValueSchemaInfo;
-
-/**
- * A wrapper for Pulsar {@link Schema}, make it serializable and can be created from {@link
- * SchemaInfo}.
- *
- * <p>General pulsar schema info (avro, json, protobuf and keyvalue) don't contain the required
- * class info. We have to urge users to provide the related type class and encode it into schema
- * info.
- */
-@Internal
-public final class PulsarSchema<T> implements Serializable {
- private static final long serialVersionUID = -2561088131419607555L;
-
- private transient Schema<T> schema;
- private transient SchemaInfo schemaInfo;
-
- /** Create serializable pulsar schema for primitive types. */
- public PulsarSchema(Schema<T> schema) {
- SchemaInfo info = schema.getSchemaInfo();
- SchemaType type = info.getType();
- checkArgument(type != SchemaType.JSON, "Json Schema should provide the type class");
- checkArgument(type != SchemaType.AVRO, "Avro Schema should provide the type class");
- checkArgument(type != SchemaType.PROTOBUF, "Protobuf Schema should provide the type class");
- checkArgument(
- type != SchemaType.PROTOBUF_NATIVE,
- "Protobuf Native Schema should provide the type class");
- checkArgument(
- type != SchemaType.KEY_VALUE,
- "Key Value Schema should provide the type class of key and value");
- // Primitive type information could be reflected from the schema class.
- Class<?> typeClass = getTemplateType1(schema.getClass());
-
- this.schemaInfo = encodeClassInfo(info, typeClass);
- this.schema = createSchema(schemaInfo);
- }
-
- /**
- * Create serializable pulsar schema for struct type or primitive types.
- *
- * @param schema The schema instance.
- * @param typeClass The type class of this schema.
- */
- public PulsarSchema(Schema<T> schema, Class<T> typeClass) {
- SchemaInfo info = schema.getSchemaInfo();
- checkArgument(
- info.getType() != SchemaType.KEY_VALUE,
- "Key Value Schema should provide the type classes of key and value");
- validateSchemaInfo(info);
-
- this.schemaInfo = encodeClassInfo(info, typeClass);
- this.schema = createSchema(schemaInfo);
- }
-
- /** Create serializable pulsar schema for key value type. */
- public <K, V> PulsarSchema(
- Schema<KeyValue<K, V>> kvSchema, Class<K> keyClass, Class<V> valueClass) {
- SchemaInfo info = kvSchema.getSchemaInfo();
- checkArgument(
- info.getType() == SchemaType.KEY_VALUE,
- "This constructor could only be applied for KeyValueSchema");
-
- KeyValue<SchemaInfo, SchemaInfo> infoKeyValue = decodeKeyValueSchemaInfo(info);
-
- SchemaInfo infoKey = encodeClassInfo(infoKeyValue.getKey(), keyClass);
- validateSchemaInfo(infoKey);
-
- SchemaInfo infoValue = encodeClassInfo(infoKeyValue.getValue(), valueClass);
- validateSchemaInfo(infoValue);
-
- KeyValueEncodingType encodingType = decodeKeyValueEncodingType(info);
- SchemaInfo encodedInfo =
- encodeKeyValueSchemaInfo(info.getName(), infoKey, infoValue, encodingType);
-
- this.schemaInfo = encodeClassInfo(encodedInfo, KeyValue.class);
- this.schema = createSchema(this.schemaInfo);
- }
-
- public Schema<T> getPulsarSchema() {
- return schema;
- }
-
- public SchemaInfo getSchemaInfo() {
- return schemaInfo;
- }
-
- public Class<T> getRecordClass() {
- return decodeClassInfo(schemaInfo);
- }
-
- private void writeObject(ObjectOutputStream oos) throws IOException {
- // Name
- oos.writeUTF(schemaInfo.getName());
-
- // Schema
- byte[] schemaBytes = schemaInfo.getSchema();
- oos.writeInt(schemaBytes.length);
- oos.write(schemaBytes);
-
- // Type
- SchemaType type = schemaInfo.getType();
- oos.writeInt(type.getValue());
-
- // Properties
- Map<String, String> properties = schemaInfo.getProperties();
- oos.writeInt(properties.size());
- for (Map.Entry<String, String> entry : properties.entrySet()) {
- oos.writeUTF(entry.getKey());
- oos.writeUTF(entry.getValue());
- }
-
- // Timestamp
- oos.writeLong(schemaInfo.getTimestamp());
- }
-
- private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
- // Name
- String name = ois.readUTF();
-
- // Schema
- int byteLen = ois.readInt();
- byte[] schemaBytes = new byte[byteLen];
- ois.readFully(schemaBytes);
-
- // Type
- int typeIdx = ois.readInt();
- SchemaType type = SchemaType.valueOf(typeIdx);
-
- // Properties
- int propSize = ois.readInt();
- Map<String, String> properties = new HashMap<>(propSize);
- for (int i = 0; i < propSize; i++) {
- properties.put(ois.readUTF(), ois.readUTF());
- }
-
- // Timestamp
- long timestamp = ois.readLong();
-
- this.schemaInfo =
- SchemaInfoImpl.builder()
- .name(name)
- .schema(schemaBytes)
- .type(type)
- .properties(properties)
- .timestamp(timestamp)
- .build();
- this.schema = createSchema(schemaInfo);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- SchemaInfo that = ((PulsarSchema<?>) o).getPulsarSchema().getSchemaInfo();
-
- return Objects.equals(schemaInfo.getType(), that.getType())
- && Arrays.equals(schemaInfo.getSchema(), that.getSchema())
- && Objects.equals(schemaInfo.getProperties(), that.getProperties());
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(
- schemaInfo.getType(),
- Arrays.hashCode(schemaInfo.getSchema()),
- schemaInfo.getProperties());
- }
-
- @Override
- public String toString() {
- return schemaInfo.toString();
- }
-
- /**
- * We would throw exception if the schema type is protobuf and users don't provide protobuf-java
- * in the class path.
- */
- private void validateSchemaInfo(SchemaInfo info) {
- SchemaType type = info.getType();
- if (type == SchemaType.PROTOBUF || type == SchemaType.PROTOBUF_NATIVE) {
- checkState(
- haveProtobuf(), "protobuf-java should be provided if you use related schema.");
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaFactory.java
deleted file mode 100644
index 15540fc6b04..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-/**
- * The schema factory for a specified {@link SchemaType}. We add this factory because of pulsar
- * don't provide a serializable schema and we can't create it directly from {@link SchemaInfo}. So
- * we have to implement this creation logic.
- */
-@Internal
-public interface PulsarSchemaFactory<T> {
-
- /** The supported schema type for this factory. We would classify the factory by the type. */
- SchemaType type();
-
- /** Create the schema by the given info. */
- Schema<T> createSchema(SchemaInfo info);
-
- /** Create the flink type information by the given schema info. */
- TypeInformation<T> createTypeInfo(SchemaInfo info);
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeInformation.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeInformation.java
deleted file mode 100644
index ef616f09503..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeInformation.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-
-import java.util.Objects;
-
-/** Wrap the pulsar {@code Schema} into a flink {@code TypeInformation}. */
-@Internal
-public class PulsarSchemaTypeInformation<T> extends TypeInformation<T> {
- private static final long serialVersionUID = 7284667318651333519L;
-
- private final PulsarSchema<T> schema;
-
- public PulsarSchemaTypeInformation(PulsarSchema<T> schema) {
- this.schema = schema;
- }
-
- @Override
- public boolean isBasicType() {
- return false;
- }
-
- @Override
- public boolean isTupleType() {
- return false;
- }
-
- @Override
- public int getArity() {
- return 1;
- }
-
- @Override
- public int getTotalFields() {
- return 1;
- }
-
- @Override
- public Class<T> getTypeClass() {
- return schema.getRecordClass();
- }
-
- @Override
- public boolean isKeyType() {
- return false;
- }
-
- @Override
- public TypeSerializer<T> createSerializer(ExecutionConfig config) {
- return new PulsarSchemaTypeSerializer<>(schema);
- }
-
- @Override
- public String toString() {
- return schema.toString();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof PulsarSchemaTypeInformation) {
- PulsarSchemaTypeInformation<?> that = (PulsarSchemaTypeInformation<?>) obj;
- return Objects.equals(schema, that.schema);
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return schema.hashCode();
- }
-
- @Override
- public boolean canEqual(Object obj) {
- return obj instanceof PulsarSchemaTypeInformation;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeSerializer.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeSerializer.java
deleted file mode 100644
index 3c58c83473e..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaTypeSerializer.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.util.InstantiationUtil;
-
-import com.google.protobuf.Message;
-import org.apache.pulsar.client.api.Schema;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Objects;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.haveProtobuf;
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.isProtobufTypeClass;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/** Wrap the pulsar {@code Schema} into a flink {@code TypeSerializer}. */
-@Internal
-public class PulsarSchemaTypeSerializer<T> extends TypeSerializer<T> {
- private static final long serialVersionUID = 7771153330969433085L;
-
- private final PulsarSchema<T> schema;
-
- public PulsarSchemaTypeSerializer(PulsarSchema<T> schema) {
- this.schema = schema;
- }
-
- @Override
- public boolean isImmutableType() {
- return false;
- }
-
- @Override
- public TypeSerializer<T> duplicate() {
- return this;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public T createInstance() {
- Class<T> recordClass = schema.getRecordClass();
-
- // No exception wouldn't be thrown here if user don't provide protobuf-java.
- if (haveProtobuf() && isProtobufTypeClass(recordClass)) {
- try {
- Method newBuilderMethod = recordClass.getMethod("newBuilder");
- Message.Builder builder = (Message.Builder) newBuilderMethod.invoke(null);
- return (T) builder.build();
- } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
- throw new IllegalStateException(e);
- }
- } else {
- return InstantiationUtil.instantiate(recordClass);
- }
- }
-
- @Override
- public T copy(T from) {
- return from;
- }
-
- @Override
- public T copy(T from, T reuse) {
- return from;
- }
-
- @Override
- public int getLength() {
- return 0;
- }
-
- @Override
- public void serialize(T record, DataOutputView target) throws IOException {
- Schema<T> pulsarSchema = schema.getPulsarSchema();
- byte[] bytes = pulsarSchema.encode(record);
-
- target.writeInt(bytes.length);
- target.write(bytes);
- }
-
- @Override
- public T deserialize(DataInputView source) throws IOException {
- int len = source.readInt();
- byte[] bytes = new byte[len];
- int readLen = source.read(bytes);
- checkState(len == readLen);
-
- Schema<T> pulsarSchema = schema.getPulsarSchema();
- return pulsarSchema.decode(bytes);
- }
-
- @Override
- public T deserialize(T reuse, DataInputView source) throws IOException {
- return deserialize(source);
- }
-
- @Override
- public void copy(DataInputView source, DataOutputView target) throws IOException {
- int len = source.readInt();
- byte[] bytes = new byte[len];
- int readLen = source.read(bytes);
- checkState(len == readLen);
-
- target.writeInt(bytes.length);
- target.write(bytes);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof PulsarSchemaTypeSerializer) {
- PulsarSchemaTypeSerializer<?> that = (PulsarSchemaTypeSerializer<?>) obj;
- return Objects.equals(schema, that.schema);
- }
-
- return false;
- }
-
- @Override
- public int hashCode() {
- return schema.hashCode();
- }
-
- @Override
- public TypeSerializerSnapshot<T> snapshotConfiguration() {
- return new PulsarSchemaTypeSerializerSnapshot<>(schema);
- }
-
- /**
- * Snapshot for PulsarSchemaTypeSerializer, we only snapshot the SerializablePulsarSchema into
- * the state.
- */
- public static final class PulsarSchemaTypeSerializerSnapshot<T>
- implements TypeSerializerSnapshot<T> {
-
- private PulsarSchema<T> schema;
-
- public PulsarSchemaTypeSerializerSnapshot(PulsarSchema<T> schema) {
- this.schema = schema;
- }
-
- @Override
- public int getCurrentVersion() {
- return 1;
- }
-
- @Override
- public void writeSnapshot(DataOutputView out) throws IOException {
- byte[] bytes = InstantiationUtil.serializeObject(schema);
- out.writeInt(bytes.length);
- out.write(bytes);
- }
-
- @Override
- public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader)
- throws IOException {
- int len = in.readInt();
- byte[] bytes = new byte[len];
- int readLen = in.read(bytes);
- checkState(readLen == len);
-
- try {
- ClassLoader loader = Thread.currentThread().getContextClassLoader();
- this.schema = InstantiationUtil.deserializeObject(bytes, loader);
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public TypeSerializer<T> restoreSerializer() {
- return new PulsarSchemaTypeSerializer<>(schema);
- }
-
- @Override
- public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(
- TypeSerializer<T> newSerializer) {
- return TypeSerializerSchemaCompatibility.compatibleAsIs();
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaUtils.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaUtils.java
deleted file mode 100644
index 00bf2cfb23c..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/PulsarSchemaUtils.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.connector.pulsar.common.schema.factories.AvroSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.JSONSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.KeyValueSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.PrimitiveSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.ProtobufNativeSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.ProtobufSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.factories.StringSchemaFactory;
-
-import com.google.protobuf.Message;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.BooleanSchema;
-import org.apache.pulsar.client.impl.schema.ByteSchema;
-import org.apache.pulsar.client.impl.schema.BytesSchema;
-import org.apache.pulsar.client.impl.schema.DateSchema;
-import org.apache.pulsar.client.impl.schema.DoubleSchema;
-import org.apache.pulsar.client.impl.schema.FloatSchema;
-import org.apache.pulsar.client.impl.schema.InstantSchema;
-import org.apache.pulsar.client.impl.schema.IntSchema;
-import org.apache.pulsar.client.impl.schema.LocalDateSchema;
-import org.apache.pulsar.client.impl.schema.LocalDateTimeSchema;
-import org.apache.pulsar.client.impl.schema.LocalTimeSchema;
-import org.apache.pulsar.client.impl.schema.LongSchema;
-import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
-import org.apache.pulsar.client.impl.schema.ShortSchema;
-import org.apache.pulsar.client.impl.schema.TimeSchema;
-import org.apache.pulsar.client.impl.schema.TimestampSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.DATE_TYPE_INFO;
-import static org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
-import static org.apache.flink.api.common.typeinfo.Types.BOOLEAN;
-import static org.apache.flink.api.common.typeinfo.Types.BYTE;
-import static org.apache.flink.api.common.typeinfo.Types.DOUBLE;
-import static org.apache.flink.api.common.typeinfo.Types.FLOAT;
-import static org.apache.flink.api.common.typeinfo.Types.INSTANT;
-import static org.apache.flink.api.common.typeinfo.Types.INT;
-import static org.apache.flink.api.common.typeinfo.Types.LOCAL_DATE;
-import static org.apache.flink.api.common.typeinfo.Types.LOCAL_DATE_TIME;
-import static org.apache.flink.api.common.typeinfo.Types.LOCAL_TIME;
-import static org.apache.flink.api.common.typeinfo.Types.LONG;
-import static org.apache.flink.api.common.typeinfo.Types.SHORT;
-import static org.apache.flink.api.common.typeinfo.Types.SQL_TIME;
-import static org.apache.flink.api.common.typeinfo.Types.SQL_TIMESTAMP;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * Util class for pulsar schema. Register all the {@link PulsarSchemaFactory} in this class and
- * provide the {@link TypeInformation} or {@link PulsarSchema} conversion.
- */
-@Internal
-@SuppressWarnings("unchecked")
-public final class PulsarSchemaUtils {
-
- private static final Class<Message> PROTOBUF_MESSAGE_CLASS;
- // Predefined pulsar schema factory.
- private static final Map<SchemaType, PulsarSchemaFactory<?>> FACTORY_REGISTER =
- new EnumMap<>(SchemaType.class);
-
- public static final String CLASS_INFO_PLACEHOLDER = "INTERNAL.pulsar.schema.type.class.name";
-
- static {
- // Initialize protobuf message class.
- Class<Message> messageClass;
- try {
- messageClass = (Class<Message>) Class.forName("com.google.protobuf.Message");
- } catch (ClassNotFoundException e) {
- messageClass = null;
- }
- PROTOBUF_MESSAGE_CLASS = messageClass;
-
- // Struct schemas
- registerSchemaFactory(new AvroSchemaFactory<>());
- registerSchemaFactory(new JSONSchemaFactory<>());
- registerSchemaFactory(new KeyValueSchemaFactory<>());
- if (haveProtobuf()) {
- // Protobuf type should be supported only when we have protobuf-java.
- registerSchemaFactory(new ProtobufNativeSchemaFactory<>());
- registerSchemaFactory(new ProtobufSchemaFactory<>());
- }
-
- // Primitive schemas
- registerSchemaFactory(new StringSchemaFactory());
- registerSchemaFactory(
- new PrimitiveSchemaFactory<>(
- SchemaType.NONE, BytesSchema.of(), BYTE_PRIMITIVE_ARRAY_TYPE_INFO));
- registerPrimitiveFactory(BooleanSchema.of(), BOOLEAN);
- registerPrimitiveFactory(ByteSchema.of(), BYTE);
- registerPrimitiveFactory(BytesSchema.of(), BYTE_PRIMITIVE_ARRAY_TYPE_INFO);
- registerPrimitiveFactory(DateSchema.of(), DATE_TYPE_INFO);
- registerPrimitiveFactory(DoubleSchema.of(), DOUBLE);
- registerPrimitiveFactory(FloatSchema.of(), FLOAT);
- registerPrimitiveFactory(InstantSchema.of(), INSTANT);
- registerPrimitiveFactory(IntSchema.of(), INT);
- registerPrimitiveFactory(LocalDateSchema.of(), LOCAL_DATE);
- registerPrimitiveFactory(LocalDateTimeSchema.of(), LOCAL_DATE_TIME);
- registerPrimitiveFactory(LocalTimeSchema.of(), LOCAL_TIME);
- registerPrimitiveFactory(LongSchema.of(), LONG);
- registerPrimitiveFactory(ShortSchema.of(), SHORT);
- registerPrimitiveFactory(TimeSchema.of(), SQL_TIME);
- registerPrimitiveFactory(TimestampSchema.of(), SQL_TIMESTAMP);
- }
-
- private PulsarSchemaUtils() {
- // No need to create instance.
- }
-
- /** A boolean value for determine if user have protobuf-java in his class path. */
- public static boolean haveProtobuf() {
- return PROTOBUF_MESSAGE_CLASS != null;
- }
-
- /**
- * Check if the given class is a protobuf generated class. Since this class would throw NP
- * exception when you don't provide protobuf-java, use this method with {@link #haveProtobuf()}
- */
- public static <T> boolean isProtobufTypeClass(Class<T> clazz) {
- return checkNotNull(PROTOBUF_MESSAGE_CLASS).isAssignableFrom(clazz);
- }
-
- private static <T> void registerPrimitiveFactory(
- Schema<T> schema, TypeInformation<T> information) {
- registerSchemaFactory(new PrimitiveSchemaFactory<>(schema, information));
- }
-
- private static void registerSchemaFactory(PulsarSchemaFactory<?> factory) {
- FACTORY_REGISTER.put(factory.type(), factory);
- }
-
- /**
- * Pulsar has a hugh set of built-in schemas. We can create them by the given {@link
- * SchemaInfo}. This schema info is a wrapped info created by {@link PulsarSchema}.
- */
- @SuppressWarnings("unchecked")
- public static <T> Schema<T> createSchema(SchemaInfo info) {
- PulsarSchemaFactory<?> factory = FACTORY_REGISTER.get(info.getType());
- checkNotNull(factory, "This schema info %s is not supported.", info);
-
- return (Schema<T>) factory.createSchema(info);
- }
-
- /**
- * Convert the {@link SchemaInfo} into a flink manageable {@link TypeInformation}. This schema
- * info is a wrapped info created by {@link PulsarSchema}.
- */
- @SuppressWarnings("unchecked")
- public static <T> TypeInformation<T> createTypeInformation(SchemaInfo info) {
- PulsarSchemaFactory<?> factory = FACTORY_REGISTER.get(info.getType());
- checkNotNull(factory, "This schema info %s is not supported.", info);
-
- return (TypeInformation<T>) factory.createTypeInfo(info);
- }
-
- public static SchemaInfo encodeClassInfo(SchemaInfo schemaInfo, Class<?> typeClass) {
- Map<String, String> properties = new HashMap<>(schemaInfo.getProperties());
- properties.put(CLASS_INFO_PLACEHOLDER, typeClass.getName());
-
- return SchemaInfoImpl.builder()
- .name(schemaInfo.getName())
- .schema(schemaInfo.getSchema())
- .type(schemaInfo.getType())
- .properties(properties)
- .timestamp(schemaInfo.getTimestamp())
- .build();
- }
-
- @SuppressWarnings("unchecked")
- public static <T> Class<T> decodeClassInfo(SchemaInfo schemaInfo) {
- Map<String, String> properties = schemaInfo.getProperties();
- String className =
- checkNotNull(
- properties.get(CLASS_INFO_PLACEHOLDER),
- "This schema don't contain a class name.");
-
- try {
- return (Class<T>) Class.forName(className);
- } catch (ClassNotFoundException e) {
- throw new IllegalStateException("Couldn't find the schema class " + className);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/AvroSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/AvroSchemaFactory.java
deleted file mode 100644
index ab97b4102cc..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/AvroSchemaFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.schema.SchemaDefinition;
-import org.apache.pulsar.client.impl.schema.AvroSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-
-/** The schema factory for pulsar's {@link AvroSchema}. */
-public class AvroSchemaFactory<T> extends BaseStructSchemaFactory<T> {
-
- @Override
- public SchemaType type() {
- return SchemaType.AVRO;
- }
-
- @Override
- public Schema<T> createSchema(SchemaInfo info) {
- Class<T> typeClass = decodeClassInfo(info);
- SchemaDefinition<T> definition =
- SchemaDefinition.<T>builder()
- .withPojo(typeClass)
- .withProperties(info.getProperties())
- .build();
-
- return AvroSchema.of(definition);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/BaseStructSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/BaseStructSchemaFactory.java
deleted file mode 100644
index dd8450308f7..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/BaseStructSchemaFactory.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchema;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaTypeInformation;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-
-/** Implement the common createTypeInfo method for all struct schema factory. */
-public abstract class BaseStructSchemaFactory<T> implements PulsarSchemaFactory<T> {
-
- @Override
- public TypeInformation<T> createTypeInfo(SchemaInfo info) {
- Schema<T> pulsarSchema = createSchema(info);
- Class<T> typeClass = decodeClassInfo(info);
- PulsarSchema<T> schema = new PulsarSchema<>(pulsarSchema, typeClass);
-
- return new PulsarSchemaTypeInformation<>(schema);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/JSONSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/JSONSchemaFactory.java
deleted file mode 100644
index 8d1a4944d0c..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/JSONSchemaFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.JSONSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-
-/** The schema factory for pulsar's {@link JSONSchema}. */
-public class JSONSchemaFactory<T> extends BaseStructSchemaFactory<T> {
-
- @Override
- public SchemaType type() {
- return SchemaType.JSON;
- }
-
- @Override
- public Schema<T> createSchema(SchemaInfo info) {
- Class<T> typeClass = decodeClassInfo(info);
- return JSONSchema.of(typeClass, info.getProperties());
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/KeyValueSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/KeyValueSchemaFactory.java
deleted file mode 100644
index 893e7e6eeb1..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/KeyValueSchemaFactory.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchema;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaFactory;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaTypeInformation;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl;
-import org.apache.pulsar.common.schema.KeyValue;
-import org.apache.pulsar.common.schema.KeyValueEncodingType;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import java.util.Map;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.CLASS_INFO_PLACEHOLDER;
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-import static org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo.decodeKeyValueEncodingType;
-import static org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo.decodeKeyValueSchemaInfo;
-
-/** The schema factory for pulsar's {@link KeyValueSchemaImpl}. */
-public class KeyValueSchemaFactory<K, V> implements PulsarSchemaFactory<KeyValue<K, V>> {
-
- @Override
- public SchemaType type() {
- return SchemaType.KEY_VALUE;
- }
-
- @Override
- public Schema<KeyValue<K, V>> createSchema(SchemaInfo info) {
- KeyValue<SchemaInfo, SchemaInfo> kvSchemaInfo = decodeKeyValueSchemaInfo(info);
-
- Schema<K> keySchema = PulsarSchemaUtils.createSchema(kvSchemaInfo.getKey());
- Schema<V> valueSchema = PulsarSchemaUtils.createSchema(kvSchemaInfo.getValue());
- KeyValueEncodingType encodingType = decodeKeyValueEncodingType(info);
-
- Schema<KeyValue<K, V>> schema = KeyValueSchemaImpl.of(keySchema, valueSchema, encodingType);
-
- // Append extra class name into schema info properties.
- // KeyValueSchema don't have custom properties builder method, we have to use side effects.
- SchemaInfo schemaInfo = schema.getSchemaInfo();
- Map<String, String> properties = schemaInfo.getProperties();
- properties.put(CLASS_INFO_PLACEHOLDER, KeyValue.class.getName());
-
- return schema;
- }
-
- @Override
- public TypeInformation<KeyValue<K, V>> createTypeInfo(SchemaInfo info) {
- KeyValue<SchemaInfo, SchemaInfo> kvSchemaInfo = decodeKeyValueSchemaInfo(info);
-
- Schema<K> keySchema = PulsarSchemaUtils.createSchema(kvSchemaInfo.getKey());
- Class<K> keyClass = decodeClassInfo(keySchema.getSchemaInfo());
-
- Schema<V> valueSchema = PulsarSchemaUtils.createSchema(kvSchemaInfo.getValue());
- Class<V> valueClass = decodeClassInfo(valueSchema.getSchemaInfo());
-
- Schema<KeyValue<K, V>> schema = createSchema(info);
- PulsarSchema<KeyValue<K, V>> pulsarSchema =
- new PulsarSchema<>(schema, keyClass, valueClass);
-
- return new PulsarSchemaTypeInformation<>(pulsarSchema);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/PrimitiveSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/PrimitiveSchemaFactory.java
deleted file mode 100644
index c56482b0894..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/PrimitiveSchemaFactory.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaFactory;
-
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableSet;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-
-/**
- * The schema factory for pulsar's primitive types. Currently, Pulsar supports <a
- * href="https://pulsar.apache.org/docs/en/schema-understand/#primitive-type">these primitive
- * types</a>.
- */
-public class PrimitiveSchemaFactory<T> implements PulsarSchemaFactory<T> {
-
- private static final ImmutableSet<SchemaType> PRIMITIVE_SCHEMA_TYPES =
- ImmutableSet.<SchemaType>builder()
- .add(SchemaType.NONE)
- .add(SchemaType.BOOLEAN)
- .add(SchemaType.INT8)
- .add(SchemaType.INT16)
- .add(SchemaType.INT32)
- .add(SchemaType.INT64)
- .add(SchemaType.FLOAT)
- .add(SchemaType.DOUBLE)
- .add(SchemaType.BYTES)
- .add(SchemaType.STRING)
- .add(SchemaType.TIMESTAMP)
- .add(SchemaType.TIME)
- .add(SchemaType.DATE)
- .add(SchemaType.INSTANT)
- .add(SchemaType.LOCAL_DATE)
- .add(SchemaType.LOCAL_TIME)
- .add(SchemaType.LOCAL_DATE_TIME)
- .build();
-
- private final SchemaType type;
- private final Schema<T> schema;
- private final TypeInformation<T> typeInformation;
-
- public PrimitiveSchemaFactory(Schema<T> schema, TypeInformation<T> typeInformation) {
- this(schema.getSchemaInfo().getType(), schema, typeInformation);
- }
-
- public PrimitiveSchemaFactory(
- SchemaType type, Schema<T> schema, TypeInformation<T> typeInformation) {
- checkArgument(PRIMITIVE_SCHEMA_TYPES.contains(type));
-
- this.type = type;
- this.schema = schema;
- this.typeInformation = typeInformation;
- }
-
- @Override
- public SchemaType type() {
- return type;
- }
-
- @Override
- public Schema<T> createSchema(SchemaInfo info) {
- return schema;
- }
-
- @Override
- public TypeInformation<T> createTypeInfo(SchemaInfo info) {
- return typeInformation;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufNativeSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufNativeSchemaFactory.java
deleted file mode 100644
index 955e5d97c74..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufNativeSchemaFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import com.google.protobuf.GeneratedMessageV3;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-
-/** The schema factory for pulsar's {@link ProtobufNativeSchema}. */
-public class ProtobufNativeSchemaFactory<T extends GeneratedMessageV3>
- extends BaseStructSchemaFactory<T> {
-
- @Override
- public SchemaType type() {
- return SchemaType.PROTOBUF_NATIVE;
- }
-
- @Override
- public Schema<T> createSchema(SchemaInfo info) {
- Class<T> typeClass = decodeClassInfo(info);
- return ProtobufNativeSchema.of(typeClass, info.getProperties());
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufSchemaFactory.java
deleted file mode 100644
index d6342193e24..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/ProtobufSchemaFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import com.google.protobuf.GeneratedMessageV3;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.ProtobufSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-import static org.apache.flink.connector.pulsar.common.schema.PulsarSchemaUtils.decodeClassInfo;
-
-/** The schema factory for pulsar's {@link ProtobufSchema}. */
-public class ProtobufSchemaFactory<T extends GeneratedMessageV3>
- extends BaseStructSchemaFactory<T> {
-
- @Override
- public SchemaType type() {
- return SchemaType.PROTOBUF;
- }
-
- @Override
- public Schema<T> createSchema(SchemaInfo info) {
- Class<T> typeClass = decodeClassInfo(info);
- return ProtobufSchema.of(typeClass, info.getProperties());
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/StringSchemaFactory.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/StringSchemaFactory.java
deleted file mode 100644
index 311052ee548..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/schema/factories/StringSchemaFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.schema.factories;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchemaFactory;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.impl.schema.StringSchema;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.common.schema.SchemaType;
-
-/** The schema factory for pulsar's {@link StringSchema}. */
-public class StringSchemaFactory implements PulsarSchemaFactory<String> {
-
- @Override
- public SchemaType type() {
- return SchemaType.STRING;
- }
-
- @Override
- public Schema<String> createSchema(SchemaInfo info) {
- // SchemaInfo contains the string encode type.
- return StringSchema.fromSchemaInfo(info);
- }
-
- @Override
- public TypeInformation<String> createTypeInfo(SchemaInfo info) {
- return Types.STRING;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarExceptionUtils.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarExceptionUtils.java
deleted file mode 100644
index 63bd65cf0fe..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarExceptionUtils.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.utils;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.function.SupplierWithException;
-import org.apache.flink.util.function.ThrowingRunnable;
-
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.PulsarClientException;
-
-/**
- * Util class for pulsar checked exceptions. Sneaky throw {@link PulsarAdminException} and {@link
- * PulsarClientException}.
- */
-@Internal
-public final class PulsarExceptionUtils {
-
- private PulsarExceptionUtils() {
- // No public constructor.
- }
-
- public static <R extends PulsarClientException> void sneakyClient(
- ThrowingRunnable<R> runnable) {
- sneaky(runnable);
- }
-
- public static <T, R extends PulsarClientException> T sneakyClient(
- SupplierWithException<T, R> supplier) {
- return sneaky(supplier);
- }
-
- public static <R extends PulsarAdminException> void sneakyAdmin(ThrowingRunnable<R> runnable) {
- sneaky(runnable);
- }
-
- public static <T, R extends PulsarAdminException> T sneakyAdmin(
- SupplierWithException<T, R> supplier) {
- return sneaky(supplier);
- }
-
- private static <R extends Exception> void sneaky(ThrowingRunnable<R> runnable) {
- try {
- runnable.run();
- } catch (Exception r) {
- sneakyThrow(r);
- }
- }
-
- /** Catch the throwable exception and rethrow it without try catch. */
- private static <T, R extends Exception> T sneaky(SupplierWithException<T, R> supplier) {
- try {
- return supplier.get();
- } catch (Exception r) {
- sneakyThrow(r);
- }
-
- // This method wouldn't be executed.
- throw new RuntimeException("Never throw here.");
- }
-
- /** javac hack for unchecking the checked exception. */
- @SuppressWarnings("unchecked")
- public static <T extends Exception> void sneakyThrow(Exception t) throws T {
- throw (T) t;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarSerdeUtils.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarSerdeUtils.java
deleted file mode 100644
index ffe041b9bd7..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarSerdeUtils.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.utils;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.function.BiConsumerWithException;
-import org.apache.flink.util.function.FunctionWithException;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/** Util for serialize and deserialize. */
-@Internal
-public final class PulsarSerdeUtils {
-
- private PulsarSerdeUtils() {
- // No public constructor.
- }
-
- // Bytes serialization.
-
- public static void serializeBytes(DataOutputStream out, byte[] bytes) throws IOException {
- out.writeInt(bytes.length);
- out.write(bytes);
- }
-
- public static byte[] deserializeBytes(DataInputStream in) throws IOException {
- int size = in.readInt();
- byte[] bytes = new byte[size];
- in.readFully(bytes);
-
- return bytes;
- }
-
- // Common Object serialization.
-
- public static void serializeObject(DataOutputStream out, Object obj) throws IOException {
- Preconditions.checkNotNull(obj);
-
- byte[] objectBytes = InstantiationUtil.serializeObject(obj);
- serializeBytes(out, objectBytes);
- }
-
- public static <T> T deserializeObject(DataInputStream in) throws IOException {
- byte[] objectBytes = deserializeBytes(in);
- ClassLoader loader = Thread.currentThread().getContextClassLoader();
-
- try {
- return InstantiationUtil.deserializeObject(objectBytes, loader);
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-
- // Common List serialization.
-
- public static <T> void serializeList(
- DataOutputStream out,
- List<T> list,
- BiConsumerWithException<DataOutputStream, T, IOException> serializer)
- throws IOException {
- out.writeInt(list.size());
- for (T t : list) {
- serializer.accept(out, t);
- }
- }
-
- public static <T> List<T> deserializeList(
- DataInputStream in, FunctionWithException<DataInputStream, T, IOException> deserializer)
- throws IOException {
- int size = in.readInt();
- List<T> set = new ArrayList<>(size);
- for (int i = 0; i < size; i++) {
- T t = deserializer.apply(in);
- set.add(t);
- }
-
- return set;
- }
-
- // Common Set serialization.
-
- public static <T> void serializeSet(
- DataOutputStream out,
- Set<T> set,
- BiConsumerWithException<DataOutputStream, T, IOException> serializer)
- throws IOException {
- out.writeInt(set.size());
- for (T t : set) {
- serializer.accept(out, t);
- }
- }
-
- public static <T> Set<T> deserializeSet(
- DataInputStream in, FunctionWithException<DataInputStream, T, IOException> deserializer)
- throws IOException {
- int size = in.readInt();
- Set<T> set = new HashSet<>(size);
- for (int i = 0; i < size; i++) {
- T t = deserializer.apply(in);
- set.add(t);
- }
-
- return set;
- }
-
- // Common Map serialization.
-
- public static <K, V> void serializeMap(
- DataOutputStream out,
- Map<K, V> map,
- BiConsumerWithException<DataOutputStream, K, IOException> keySerializer,
- BiConsumerWithException<DataOutputStream, V, IOException> valueSerializer)
- throws IOException {
- out.writeInt(map.size());
- for (Map.Entry<K, V> entry : map.entrySet()) {
- keySerializer.accept(out, entry.getKey());
- valueSerializer.accept(out, entry.getValue());
- }
- }
-
- public static <K, V> Map<K, V> deserializeMap(
- DataInputStream in,
- FunctionWithException<DataInputStream, K, IOException> keyDeserializer,
- FunctionWithException<DataInputStream, V, IOException> valueDeserializer)
- throws IOException {
- int size = in.readInt();
- Map<K, V> result = new HashMap<>(size);
- for (int i = 0; i < size; i++) {
- K key = keyDeserializer.apply(in);
- V value = valueDeserializer.apply(in);
- result.put(key, value);
- }
- return result;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarTransactionUtils.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarTransactionUtils.java
deleted file mode 100644
index c8ffe6387fd..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/utils/PulsarTransactionUtils.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.common.utils;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.transaction.Transaction;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException;
-
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient;
-import static org.apache.flink.util.ExceptionUtils.findThrowable;
-
-/** A suit of workarounds for the Pulsar Transaction. */
-@Internal
-public final class PulsarTransactionUtils {
-
- private PulsarTransactionUtils() {
- // No public constructor
- }
-
- /** Create transaction with given timeout millis. */
- public static Transaction createTransaction(PulsarClient pulsarClient, long timeoutMs) {
- try {
- CompletableFuture<Transaction> future =
- sneakyClient(pulsarClient::newTransaction)
- .withTransactionTimeout(timeoutMs, TimeUnit.MILLISECONDS)
- .build();
-
- return future.get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IllegalStateException(e);
- } catch (ExecutionException e) {
- throw new FlinkRuntimeException(e);
- }
- }
-
- /**
- * This is a bug in original {@link TransactionCoordinatorClientException#unwrap(Throwable)}
- * method. Pulsar wraps the {@link ExecutionException} which hides the real execution exception.
- *
- * <p>This bug should be fixed after the 2.10.0 release. We just keep this for safety.
- */
- public static TransactionCoordinatorClientException unwrap(
- TransactionCoordinatorClientException e) {
- return findThrowable(e.getCause(), TransactionCoordinatorClientException.class).orElse(e);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSink.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSink.java
deleted file mode 100644
index 4c6c4a9b7c7..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSink.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.connector.sink2.Committer;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable;
-import org.apache.flink.connector.pulsar.sink.committer.PulsarCommittableSerializer;
-import org.apache.flink.connector.pulsar.sink.committer.PulsarCommitter;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.PulsarWriter;
-import org.apache.flink.connector.pulsar.sink.writer.delayer.MessageDelayer;
-import org.apache.flink.connector.pulsar.sink.writer.router.KeyHashTopicRouter;
-import org.apache.flink.connector.pulsar.sink.writer.router.RoundRobinTopicRouter;
-import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
-import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
-import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
-import org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * The Sink implementation of Pulsar. Please use a {@link PulsarSinkBuilder} to construct a {@link
- * PulsarSink}. The following example shows how to create a PulsarSink receiving records of {@code
- * String} type.
- *
- * <pre>{@code
- * PulsarSink<String> sink = PulsarSink.builder()
- * .setServiceUrl(operator().serviceUrl())
- * .setAdminUrl(operator().adminUrl())
- * .setTopic(topic)
- * .setSerializationSchema(PulsarSerializationSchema.pulsarSchema(Schema.STRING))
- * .build();
- * }</pre>
- *
- * <p>The sink supports all delivery guarantees described by {@link DeliveryGuarantee}.
- *
- * <ul>
- * <li>{@link DeliveryGuarantee#NONE} does not provide any guarantees: messages may be lost in
- * case of issues on the Pulsar broker and messages may be duplicated in case of a Flink
- * failure.
- * <li>{@link DeliveryGuarantee#AT_LEAST_ONCE} the sink will wait for all outstanding records in
- * the Pulsar buffers to be acknowledged by the Pulsar producer on a checkpoint. No messages
- * will be lost in case of any issue with the Pulsar brokers but messages may be duplicated
- * when Flink restarts.
- * <li>{@link DeliveryGuarantee#EXACTLY_ONCE}: In this mode the PulsarSink will write all messages
- * in a Pulsar transaction that will be committed to Pulsar on a checkpoint. Thus, no
- * duplicates will be seen in case of a Flink restart. However, this delays record writing
- * effectively until a checkpoint is written, so adjust the checkpoint duration accordingly.
- * Additionally, it is highly recommended to tweak Pulsar transaction timeout (link) >>
- * maximum checkpoint duration + maximum restart duration or data loss may happen when Pulsar
- * expires an uncommitted transaction.
- * </ul>
- *
- * <p>See {@link PulsarSinkBuilder} for more details.
- *
- * @param <IN> The input type of the sink.
- */
-@PublicEvolving
-public class PulsarSink<IN> implements TwoPhaseCommittingSink<IN, PulsarCommittable> {
- private static final long serialVersionUID = 4416714587951282119L;
-
- private final SinkConfiguration sinkConfiguration;
- private final PulsarSerializationSchema<IN> serializationSchema;
- private final TopicMetadataListener metadataListener;
- private final MessageDelayer<IN> messageDelayer;
- private final TopicRouter<IN> topicRouter;
-
- PulsarSink(
- SinkConfiguration sinkConfiguration,
- PulsarSerializationSchema<IN> serializationSchema,
- TopicMetadataListener metadataListener,
- TopicRoutingMode topicRoutingMode,
- TopicRouter<IN> topicRouter,
- MessageDelayer<IN> messageDelayer) {
- this.sinkConfiguration = checkNotNull(sinkConfiguration);
- this.serializationSchema = checkNotNull(serializationSchema);
- this.metadataListener = checkNotNull(metadataListener);
- this.messageDelayer = checkNotNull(messageDelayer);
- checkNotNull(topicRoutingMode);
-
- // Create topic router supplier.
- if (topicRoutingMode == TopicRoutingMode.CUSTOM) {
- this.topicRouter = checkNotNull(topicRouter);
- } else if (topicRoutingMode == TopicRoutingMode.ROUND_ROBIN) {
- this.topicRouter = new RoundRobinTopicRouter<>(sinkConfiguration);
- } else {
- this.topicRouter = new KeyHashTopicRouter<>(sinkConfiguration);
- }
- }
-
- /**
- * Create a {@link PulsarSinkBuilder} to construct a new {@link PulsarSink}.
- *
- * @param <IN> Type of incoming records.
- * @return A Pulsar sink builder.
- */
- public static <IN> PulsarSinkBuilder<IN> builder() {
- return new PulsarSinkBuilder<>();
- }
-
- @Internal
- @Override
- public PrecommittingSinkWriter<IN, PulsarCommittable> createWriter(InitContext initContext) {
- return new PulsarWriter<>(
- sinkConfiguration,
- serializationSchema,
- metadataListener,
- topicRouter,
- messageDelayer,
- initContext);
- }
-
- @Internal
- @Override
- public Committer<PulsarCommittable> createCommitter() {
- return new PulsarCommitter(sinkConfiguration);
- }
-
- @Internal
- @Override
- public SimpleVersionedSerializer<PulsarCommittable> getCommittableSerializer() {
- return new PulsarCommittableSerializer();
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkBuilder.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkBuilder.java
deleted file mode 100644
index 4e8f9b28f0f..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkBuilder.java
+++ /dev/null
@@ -1,410 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.common.config.PulsarConfigBuilder;
-import org.apache.flink.connector.pulsar.common.config.PulsarOptions;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.delayer.MessageDelayer;
-import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
-import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
-import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSchemaWrapper;
-import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
-import org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener;
-
-import org.apache.pulsar.client.api.Schema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAMS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAM_MAP;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PLUGIN_CLASS_NAME;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_TRANSACTION;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_PRODUCER_NAME;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_SEND_TIMEOUT_MS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_DELIVERY_GUARANTEE;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_SCHEMA_EVOLUTION;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_TRANSACTION_TIMEOUT;
-import static org.apache.flink.connector.pulsar.sink.config.PulsarSinkConfigUtils.SINK_CONFIG_VALIDATOR;
-import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.distinctTopics;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * The builder class for {@link PulsarSink} to make it easier for the users to construct a {@link
- * PulsarSink}.
- *
- * <p>The following example shows the minimum setup to create a PulsarSink that reads the String
- * values from a Pulsar topic.
- *
- * <pre>{@code
- * PulsarSink<String> sink = PulsarSink.builder()
- * .setServiceUrl(operator().serviceUrl())
- * .setAdminUrl(operator().adminUrl())
- * .setTopics(topic)
- * .setSerializationSchema(PulsarSerializationSchema.pulsarSchema(Schema.STRING))
- * .build();
- * }</pre>
- *
- * <p>The service url, admin url, and the record serializer are required fields that must be set. If
- * you don't set the topics, make sure you have provided a custom {@link TopicRouter}. Otherwise,
- * you must provide the topics to produce.
- *
- * <p>To specify the delivery guarantees of PulsarSink, one can call {@link
- * #setDeliveryGuarantee(DeliveryGuarantee)}. The default value of the delivery guarantee is {@link
- * DeliveryGuarantee#NONE}, and it wouldn't promise the consistence when write the message into
- * Pulsar.
- *
- * <pre>{@code
- * PulsarSink<String> sink = PulsarSink.builder()
- * .setServiceUrl(operator().serviceUrl())
- * .setAdminUrl(operator().adminUrl())
- * .setTopics(topic)
- * .setSerializationSchema(PulsarSerializationSchema.pulsarSchema(Schema.STRING))
- * .setDeliveryGuarantee(deliveryGuarantee)
- * .build();
- * }</pre>
- *
- * @see PulsarSink for a more detailed explanation of the different guarantees.
- * @param <IN> The input type of the sink.
- */
-@PublicEvolving
-public class PulsarSinkBuilder<IN> {
- private static final Logger LOG = LoggerFactory.getLogger(PulsarSinkBuilder.class);
-
- private final PulsarConfigBuilder configBuilder;
-
- private PulsarSerializationSchema<IN> serializationSchema;
- private TopicMetadataListener metadataListener;
- private TopicRoutingMode topicRoutingMode;
- private TopicRouter<IN> topicRouter;
- private MessageDelayer<IN> messageDelayer;
-
- // private builder constructor.
- PulsarSinkBuilder() {
- this.configBuilder = new PulsarConfigBuilder();
- }
-
- /**
- * Sets the admin endpoint for the PulsarAdmin of the PulsarSink.
- *
- * @param adminUrl The url for the PulsarAdmin.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setAdminUrl(String adminUrl) {
- return setConfig(PULSAR_ADMIN_URL, adminUrl);
- }
-
- /**
- * Sets the server's link for the PulsarProducer of the PulsarSink.
- *
- * @param serviceUrl The server url of the Pulsar cluster.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setServiceUrl(String serviceUrl) {
- return setConfig(PULSAR_SERVICE_URL, serviceUrl);
- }
-
- /**
- * The producer name is informative, and it can be used to identify a particular producer
- * instance from the topic stats.
- *
- * @param producerName The name of the producer used in Pulsar sink.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setProducerName(String producerName) {
- return setConfig(PULSAR_PRODUCER_NAME, producerName);
- }
-
- /**
- * Set a pulsar topic list for flink sink. Some topic may not exist currently, write to this
- * non-existed topic wouldn't throw any exception.
- *
- * @param topics The topic list you would like to consume message.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setTopics(String... topics) {
- return setTopics(Arrays.asList(topics));
- }
-
- /**
- * Set a pulsar topic list for flink sink. Some topic may not exist currently, consuming this
- * non-existed topic wouldn't throw any exception.
- *
- * @param topics The topic list you would like to consume message.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setTopics(List<String> topics) {
- checkState(metadataListener == null, "setTopics couldn't be set twice.");
- // Making sure the topic should be distinct.
- List<String> topicSet = distinctTopics(topics);
- this.metadataListener = new TopicMetadataListener(topicSet);
- return this;
- }
-
- /**
- * Sets the wanted the {@link DeliveryGuarantee}. The default delivery guarantee is {@link
- * DeliveryGuarantee#NONE}.
- *
- * @param deliveryGuarantee Deliver guarantees.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setDeliveryGuarantee(DeliveryGuarantee deliveryGuarantee) {
- checkNotNull(deliveryGuarantee, "deliveryGuarantee");
- configBuilder.override(PULSAR_WRITE_DELIVERY_GUARANTEE, deliveryGuarantee);
- return this;
- }
-
- /**
- * Set a routing mode for choosing right topic partition to send messages.
- *
- * @param topicRoutingMode Routing policy for choosing the desired topic.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setTopicRoutingMode(TopicRoutingMode topicRoutingMode) {
- checkArgument(
- topicRoutingMode != TopicRoutingMode.CUSTOM,
- "CUSTOM mode should be set by using setTopicRouter method.");
- this.topicRoutingMode = checkNotNull(topicRoutingMode, "topicRoutingMode");
- return this;
- }
-
- /**
- * Use a custom topic router instead predefine topic routing.
- *
- * @param topicRouter The router for choosing topic to send message.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setTopicRouter(TopicRouter<IN> topicRouter) {
- if (topicRoutingMode != null && topicRoutingMode != TopicRoutingMode.CUSTOM) {
- LOG.warn("We would override topicRoutingMode to CUSTOM if you provide TopicRouter.");
- }
- this.topicRoutingMode = TopicRoutingMode.CUSTOM;
- this.topicRouter = checkNotNull(topicRouter, "topicRouter");
- return this;
- }
-
- /**
- * Sets the {@link PulsarSerializationSchema} that transforms incoming records to bytes.
- *
- * @param serializationSchema Pulsar specified serialize logic.
- * @return this PulsarSinkBuilder.
- */
- public <T extends IN> PulsarSinkBuilder<T> setSerializationSchema(
- PulsarSerializationSchema<T> serializationSchema) {
- PulsarSinkBuilder<T> self = specialized();
- self.serializationSchema = serializationSchema;
- return self;
- }
-
- /**
- * If you enable this option, we would consume and deserialize the message by using Pulsar
- * {@link Schema}.
- *
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> enableSchemaEvolution() {
- configBuilder.override(PULSAR_WRITE_SCHEMA_EVOLUTION, true);
- return this;
- }
-
- /**
- * Set a message delayer for enable Pulsar message delay delivery.
- *
- * @param messageDelayer The delayer which would defined when to send the message to consumer.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> delaySendingMessage(MessageDelayer<IN> messageDelayer) {
- this.messageDelayer = checkNotNull(messageDelayer);
- return this;
- }
-
- /**
- * Configure the authentication provider to use in the Pulsar client instance.
- *
- * @param authPluginClassName name of the Authentication-Plugin you want to use
- * @param authParamsString string which represents parameters for the Authentication-Plugin,
- * e.g., "key1:val1,key2:val2"
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setAuthentication(
- String authPluginClassName, String authParamsString) {
- configBuilder.set(PULSAR_AUTH_PLUGIN_CLASS_NAME, authPluginClassName);
- configBuilder.set(PULSAR_AUTH_PARAMS, authParamsString);
- return this;
- }
-
- /**
- * Configure the authentication provider to use in the Pulsar client instance.
- *
- * @param authPluginClassName name of the Authentication-Plugin you want to use
- * @param authParams map which represents parameters for the Authentication-Plugin
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setAuthentication(
- String authPluginClassName, Map<String, String> authParams) {
- configBuilder.set(PULSAR_AUTH_PLUGIN_CLASS_NAME, authPluginClassName);
- configBuilder.set(PULSAR_AUTH_PARAM_MAP, authParams);
- return this;
- }
-
- /**
- * Set an arbitrary property for the PulsarSink and Pulsar Producer. The valid keys can be found
- * in {@link PulsarSinkOptions} and {@link PulsarOptions}.
- *
- * <p>Make sure the option could be set only once or with same value.
- *
- * @param key The key of the property.
- * @param value The value of the property.
- * @return this PulsarSinkBuilder.
- */
- public <T> PulsarSinkBuilder<IN> setConfig(ConfigOption<T> key, T value) {
- configBuilder.set(key, value);
- return this;
- }
-
- /**
- * Set arbitrary properties for the PulsarSink and Pulsar Producer. The valid keys can be found
- * in {@link PulsarSinkOptions} and {@link PulsarOptions}.
- *
- * @param config The config to set for the PulsarSink.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setConfig(Configuration config) {
- configBuilder.set(config);
- return this;
- }
-
- /**
- * Set arbitrary properties for the PulsarSink and Pulsar Producer. The valid keys can be found
- * in {@link PulsarSinkOptions} and {@link PulsarOptions}.
- *
- * <p>This method is mainly used for future flink SQL binding.
- *
- * @param properties The config properties to set for the PulsarSink.
- * @return this PulsarSinkBuilder.
- */
- public PulsarSinkBuilder<IN> setProperties(Properties properties) {
- configBuilder.set(properties);
- return this;
- }
-
- /**
- * Build the {@link PulsarSink}.
- *
- * @return a PulsarSink with the settings made for this builder.
- */
- public PulsarSink<IN> build() {
- // Change delivery guarantee.
- DeliveryGuarantee deliveryGuarantee = configBuilder.get(PULSAR_WRITE_DELIVERY_GUARANTEE);
- if (deliveryGuarantee == DeliveryGuarantee.NONE) {
- LOG.warn(
- "You haven't set delivery guarantee or set it to NONE, this would cause data loss. Make sure you have known this shortcoming.");
- } else if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- LOG.info(
- "Exactly once require flink checkpoint and your pulsar cluster should support the transaction.");
- configBuilder.override(PULSAR_ENABLE_TRANSACTION, true);
- configBuilder.override(PULSAR_SEND_TIMEOUT_MS, 0L);
-
- if (!configBuilder.contains(PULSAR_WRITE_TRANSACTION_TIMEOUT)) {
- LOG.warn(
- "The default pulsar transaction timeout is 3 hours, make sure it was greater than your checkpoint interval.");
- } else {
- Long timeout = configBuilder.get(PULSAR_WRITE_TRANSACTION_TIMEOUT);
- LOG.warn(
- "The configured transaction timeout is {} mille seconds, make sure it was greater than your checkpoint interval.",
- timeout);
- }
- }
-
- if (!configBuilder.contains(PULSAR_PRODUCER_NAME)) {
- LOG.warn(
- "We recommend set a readable producer name through setProducerName(String) in production mode.");
- } else {
- String producerName = configBuilder.get(PULSAR_PRODUCER_NAME);
- if (!producerName.contains("%s")) {
- configBuilder.override(PULSAR_PRODUCER_NAME, producerName + " - %s");
- }
- }
-
- checkNotNull(serializationSchema, "serializationSchema must be set.");
- if (serializationSchema instanceof PulsarSchemaWrapper
- && !Boolean.TRUE.equals(configBuilder.get(PULSAR_WRITE_SCHEMA_EVOLUTION))) {
- LOG.info(
- "It seems like you want to send message in Pulsar Schema."
- + " You can enableSchemaEvolution for using this feature."
- + " We would use Schema.BYTES as the default schema if you don't enable this option.");
- }
-
- // Topic metadata listener validation.
- if (metadataListener == null) {
- if (topicRouter == null) {
- throw new NullPointerException(
- "No topic names or custom topic router are provided.");
- } else {
- LOG.warn(
- "No topic set has been provided, make sure your custom topic router support empty topic set.");
- this.metadataListener = new TopicMetadataListener();
- }
- }
-
- // Topic routing mode validate.
- if (topicRoutingMode == null) {
- LOG.info("No topic routing mode has been chosen. We use round-robin mode as default.");
- this.topicRoutingMode = TopicRoutingMode.ROUND_ROBIN;
- }
-
- if (messageDelayer == null) {
- this.messageDelayer = MessageDelayer.never();
- }
-
- // This is an unmodifiable configuration for Pulsar.
- // We don't use Pulsar's built-in configure classes for compatible requirement.
- SinkConfiguration sinkConfiguration =
- configBuilder.build(SINK_CONFIG_VALIDATOR, SinkConfiguration::new);
-
- return new PulsarSink<>(
- sinkConfiguration,
- serializationSchema,
- metadataListener,
- topicRoutingMode,
- topicRouter,
- messageDelayer);
- }
-
- // ------------- private helpers --------------
-
- /** Helper method for java compiler recognize the generic type. */
- @SuppressWarnings("unchecked")
- private <T extends IN> PulsarSinkBuilder<T> specialized() {
- return (PulsarSinkBuilder<T>) this;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkOptions.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkOptions.java
deleted file mode 100644
index 39b5f7301db..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/PulsarSinkOptions.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.docs.ConfigGroup;
-import org.apache.flink.annotation.docs.ConfigGroups;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-import org.apache.flink.configuration.description.Description;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.common.config.PulsarOptions;
-import org.apache.flink.connector.pulsar.sink.writer.router.MessageKeyHash;
-
-import org.apache.pulsar.client.api.CompressionType;
-
-import java.time.Duration;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.flink.configuration.description.LinkElement.link;
-import static org.apache.flink.configuration.description.TextElement.code;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MEMORY_LIMIT_BYTES;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PRODUCER_CONFIG_PREFIX;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.SINK_CONFIG_PREFIX;
-import static org.apache.flink.connector.pulsar.sink.writer.router.MessageKeyHash.MURMUR3_32_HASH;
-import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES;
-
-/**
- * Configurations for PulsarSink. All the options list here could be configured in {@link
- * PulsarSinkBuilder#setConfig(ConfigOption, Object)}. The {@link PulsarOptions} is also required
- * for pulsar source.
- *
- * @see PulsarOptions for shared configure options.
- */
-@PublicEvolving
-@ConfigGroups(
- groups = {
- @ConfigGroup(name = "PulsarSink", keyPrefix = SINK_CONFIG_PREFIX),
- @ConfigGroup(name = "PulsarProducer", keyPrefix = PRODUCER_CONFIG_PREFIX)
- })
-public final class PulsarSinkOptions {
-
- // Pulsar sink connector config prefix.
- public static final String SINK_CONFIG_PREFIX = "pulsar.sink.";
- // Pulsar producer API config prefix.
- public static final String PRODUCER_CONFIG_PREFIX = "pulsar.producer.";
-
- private PulsarSinkOptions() {
- // This is a constant class
- }
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // The configuration for pulsar sink part.
- // All the configuration listed below should have the pulsar.sink prefix.
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- public static final ConfigOption<DeliveryGuarantee> PULSAR_WRITE_DELIVERY_GUARANTEE =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "deliveryGuarantee")
- .enumType(DeliveryGuarantee.class)
- .defaultValue(DeliveryGuarantee.NONE)
- .withDescription("Optional delivery guarantee when committing.");
-
- public static final ConfigOption<Long> PULSAR_WRITE_TRANSACTION_TIMEOUT =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "transactionTimeoutMillis")
- .longType()
- .defaultValue(Duration.ofHours(3).toMillis())
- .withDescription(
- Description.builder()
- .text(
- "This option is used when the user require the %s semantic.",
- code("DeliveryGuarantee.EXACTLY_ONCE"))
- .text(
- "We would use transaction for making sure the message could be write only once.")
- .build());
-
- public static final ConfigOption<Long> PULSAR_TOPIC_METADATA_REFRESH_INTERVAL =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "topicMetadataRefreshInterval")
- .longType()
- .defaultValue(Duration.ofMinutes(30).toMillis())
- .withDescription(
- "Auto update the topic metadata in a fixed interval (in ms). The default value is 30 minutes.");
-
- public static final ConfigOption<MessageKeyHash> PULSAR_MESSAGE_KEY_HASH =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "messageKeyHash")
- .enumType(MessageKeyHash.class)
- .defaultValue(MURMUR3_32_HASH)
- .withDescription(
- "The hash policy for routing message by calculating the hash code of message key.");
-
- public static final ConfigOption<Boolean> PULSAR_WRITE_SCHEMA_EVOLUTION =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "enableSchemaEvolution")
- .booleanType()
- .defaultValue(false)
- .withDescription(
- Description.builder()
- .text(
- "If you enable this option and use PulsarSerializationSchema.pulsarSchema(),"
- + " we would consume and deserialize the message by using Pulsar's %s.",
- code("Schema"))
- .build());
-
- public static final ConfigOption<Integer> PULSAR_MAX_RECOMMIT_TIMES =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "maxRecommitTimes")
- .intType()
- .defaultValue(5)
- .withDescription(
- "The allowed transaction recommit times if we meet some retryable exception."
- + " This is used in Pulsar Transaction.");
-
- /** @deprecated This config option was removed for better performance. */
- @Deprecated
- public static final ConfigOption<Integer> PULSAR_MAX_PENDING_MESSAGES_ON_PARALLELISM =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "maxPendingMessages")
- .intType()
- .defaultValue(1000)
- .withDescription(
- Description.builder()
- .text(
- "The maximum number of pending messages in one sink parallelism.")
- .build());
-
- public static final ConfigOption<Boolean> PULSAR_ENABLE_SINK_METRICS =
- ConfigOptions.key(SINK_CONFIG_PREFIX + "enableMetrics")
- .booleanType()
- .defaultValue(true)
- .withDescription(
- Description.builder()
- .text(
- "The metrics from Pulsar Producer are only exposed if you enable this option.")
- .text(
- "You should set the %s to a positive value if you enable this option.",
- code(PULSAR_STATS_INTERVAL_SECONDS.key()))
- .build());
-
- ///////////////////////////////////////////////////////////////////////////////
- //
- // The configuration for ProducerConfigurationData part.
- // All the configuration listed below should have the pulsar.producer prefix.
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- public static final ConfigOption<String> PULSAR_PRODUCER_NAME =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "producerName")
- .stringType()
- .noDefaultValue()
- .withDescription(
- "A producer name which would be displayed in the Pulsar's dashboard."
- + " If no producer name was provided, we would use a Pulsar generated name instead.");
-
- public static final ConfigOption<Long> PULSAR_SEND_TIMEOUT_MS =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "sendTimeoutMs")
- .longType()
- .defaultValue(30000L)
- .withDescription(
- Description.builder()
- .text("Message send timeout in ms.")
- .text(
- "If a message is not acknowledged by a server before the %s expires, an error occurs.",
- code("sendTimeout"))
- .build());
-
- /** @deprecated Use {@link PulsarOptions#PULSAR_MEMORY_LIMIT_BYTES} since Pulsar 2.10.0 */
- @Deprecated
- public static final ConfigOption<Integer> PULSAR_MAX_PENDING_MESSAGES =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "maxPendingMessages")
- .intType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text("The maximum size of a queue holding pending messages.")
- .linebreak()
- .text(
- "For example, a message waiting to receive an acknowledgment from a %s.",
- link(
- "broker",
- "https://pulsar.apache.org/docs/en/reference-terminology#broker"))
- .linebreak()
- .text(
- "By default, when the queue is full, all calls to the %s and %s methods fail unless you set %s to true.",
- code("Send"),
- code("SendAsync"),
- code("BlockIfQueueFull"))
- .text(
- "Since Pulsar 2.10.0, you shouldn't set this option, use %s instead.",
- code(PULSAR_MEMORY_LIMIT_BYTES.key()))
- .build());
-
- /** @deprecated Use {@link PulsarOptions#PULSAR_MEMORY_LIMIT_BYTES} since Pulsar 2.10.0 */
- @Deprecated
- public static final ConfigOption<Integer> PULSAR_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "maxPendingMessagesAcrossPartitions")
- .intType()
- .noDefaultValue()
- .withDescription(
- Description.builder()
- .text(
- "The maximum number of pending messages across partitions.")
- .linebreak()
- .text(
- "Use the setting to lower the max pending messages for each partition (%s) if the total number exceeds the configured value.",
- code("setMaxPendingMessages"))
- .text(
- "Since Pulsar 2.10.0, you shouldn't set this option, use %s instead.",
- code(PULSAR_MEMORY_LIMIT_BYTES.key()))
- .build());
-
- public static final ConfigOption<Long> PULSAR_BATCHING_MAX_PUBLISH_DELAY_MICROS =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "batchingMaxPublishDelayMicros")
- .longType()
- .defaultValue(MILLISECONDS.toMicros(1))
- .withDescription("Batching time period of sending messages.");
-
- public static final ConfigOption<Integer>
- PULSAR_BATCHING_PARTITION_SWITCH_FREQUENCY_BY_PUBLISH_DELAY =
- ConfigOptions.key(
- PRODUCER_CONFIG_PREFIX
- + "batchingPartitionSwitchFrequencyByPublishDelay")
- .intType()
- .defaultValue(10)
- .withDescription(
- "The maximum wait time for switching topic partitions.");
-
- public static final ConfigOption<Integer> PULSAR_BATCHING_MAX_MESSAGES =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "batchingMaxMessages")
- .intType()
- .defaultValue(DEFAULT_BATCHING_MAX_MESSAGES)
- .withDescription("The maximum number of messages permitted in a batch.");
-
- public static final ConfigOption<Integer> PULSAR_BATCHING_MAX_BYTES =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "batchingMaxBytes")
- .intType()
- .defaultValue(128 * 1024)
- .withDescription(
- "The maximum size of messages permitted in a batch. Keep the maximum consistent as previous versions.");
-
- public static final ConfigOption<Boolean> PULSAR_BATCHING_ENABLED =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "batchingEnabled")
- .booleanType()
- .defaultValue(true)
- .withDescription("Enable batch send ability, it was enabled by default.");
-
- public static final ConfigOption<Boolean> PULSAR_CHUNKING_ENABLED =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "chunkingEnabled")
- .booleanType()
- .defaultValue(false)
- .withDescription("");
-
- public static final ConfigOption<CompressionType> PULSAR_COMPRESSION_TYPE =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "compressionType")
- .enumType(CompressionType.class)
- .defaultValue(CompressionType.NONE)
- .withDescription(
- Description.builder()
- .text("Message data compression type used by a producer.")
- .text("Available options:")
- .list(
- link("LZ4", "https://github.com/lz4/lz4"),
- link("ZLIB", "https://zlib.net/"),
- link("ZSTD", "https://facebook.github.io/zstd/"),
- link("SNAPPY", "https://google.github.io/snappy/"))
- .build());
-
- public static final ConfigOption<Long> PULSAR_INITIAL_SEQUENCE_ID =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "initialSequenceId")
- .longType()
- .noDefaultValue()
- .withDescription(
- "The sequence id for avoiding the duplication, it's used when Pulsar doesn't have transaction.");
-
- public static final ConfigOption<Map<String, String>> PULSAR_PRODUCER_PROPERTIES =
- ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "properties")
- .mapType()
- .defaultValue(emptyMap())
- .withDescription(
- Description.builder()
- .text("A name or value property of this consumer.")
- .text(
- " %s is application defined metadata attached to a consumer.",
- code("properties"))
- .text(
- " When getting a topic stats, associate this metadata with the consumer stats for easier identification.")
- .build());
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittable.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittable.java
deleted file mode 100644
index cca8e805f46..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittable.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.committer;
-
-import org.apache.flink.annotation.Internal;
-
-import org.apache.pulsar.client.api.transaction.TxnID;
-
-import java.util.Objects;
-
-/** The writer state for Pulsar connector. We would used in Pulsar committer. */
-@Internal
-public class PulsarCommittable {
-
- /** The transaction id. */
- private final TxnID txnID;
-
- /** The topic name with partition information. */
- private final String topic;
-
- public PulsarCommittable(TxnID txnID, String topic) {
- this.txnID = txnID;
- this.topic = topic;
- }
-
- public TxnID getTxnID() {
- return txnID;
- }
-
- public String getTopic() {
- return topic;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- PulsarCommittable that = (PulsarCommittable) o;
- return Objects.equals(txnID, that.txnID) && Objects.equals(topic, that.topic);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(txnID, topic);
- }
-
- @Override
- public String toString() {
- return "PulsarCommittable{" + "txnID=" + txnID + ", topic='" + topic + '\'' + '}';
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittableSerializer.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittableSerializer.java
deleted file mode 100644
index 324a7c6dff6..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommittableSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.committer;
-
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import org.apache.pulsar.client.api.transaction.TxnID;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/** A serializer used to serialize {@link PulsarCommittable}. */
-public class PulsarCommittableSerializer implements SimpleVersionedSerializer<PulsarCommittable> {
-
- private static final int CURRENT_VERSION = 1;
-
- @Override
- public int getVersion() {
- return CURRENT_VERSION;
- }
-
- @Override
- public byte[] serialize(PulsarCommittable obj) throws IOException {
- try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final DataOutputStream out = new DataOutputStream(baos)) {
- TxnID txnID = obj.getTxnID();
- out.writeLong(txnID.getMostSigBits());
- out.writeLong(txnID.getLeastSigBits());
- out.writeUTF(obj.getTopic());
- out.flush();
- return baos.toByteArray();
- }
- }
-
- @Override
- public PulsarCommittable deserialize(int version, byte[] serialized) throws IOException {
- try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- final DataInputStream in = new DataInputStream(bais)) {
- long mostSigBits = in.readLong();
- long leastSigBits = in.readLong();
- TxnID txnID = new TxnID(mostSigBits, leastSigBits);
- String topic = in.readUTF();
- return new PulsarCommittable(txnID, topic);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommitter.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommitter.java
deleted file mode 100644
index 8389bdc5ea5..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/committer/PulsarCommitter.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.committer;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.connector.sink2.Committer;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.common.utils.PulsarTransactionUtils;
-import org.apache.flink.connector.pulsar.sink.PulsarSink;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException.CoordinatorNotFoundException;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException.InvalidTxnStatusException;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException.MetaStoreHandlerNotExistsException;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClientException.TransactionNotFoundException;
-import org.apache.pulsar.client.api.transaction.TxnID;
-import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-
-import static org.apache.flink.connector.pulsar.common.config.PulsarClientFactory.createClient;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.pulsar.common.naming.TopicName.TRANSACTION_COORDINATOR_ASSIGN;
-
-/**
- * Committer implementation for {@link PulsarSink}.
- *
- * <p>The committer is responsible to finalize the Pulsar transactions by committing them.
- */
-@Internal
-public class PulsarCommitter implements Committer<PulsarCommittable>, Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(PulsarCommitter.class);
-
- private final SinkConfiguration sinkConfiguration;
-
- private PulsarClient pulsarClient;
- private TransactionCoordinatorClient coordinatorClient;
-
- public PulsarCommitter(SinkConfiguration sinkConfiguration) {
- this.sinkConfiguration = checkNotNull(sinkConfiguration);
- }
-
- @Override
- public void commit(Collection<CommitRequest<PulsarCommittable>> requests)
- throws IOException, InterruptedException {
- TransactionCoordinatorClient client = transactionCoordinatorClient();
-
- for (CommitRequest<PulsarCommittable> request : requests) {
- PulsarCommittable committable = request.getCommittable();
- TxnID txnID = committable.getTxnID();
- String topic = committable.getTopic();
-
- LOG.debug("Start committing the Pulsar transaction {} for topic {}", txnID, topic);
- try {
- client.commit(txnID);
- } catch (TransactionCoordinatorClientException e) {
- // This is a known bug for Pulsar Transaction.
- // We have to use instanceof instead of catching them.
- TransactionCoordinatorClientException ex = PulsarTransactionUtils.unwrap(e);
- if (ex instanceof CoordinatorNotFoundException) {
- LOG.error(
- "We couldn't find the Transaction Coordinator from Pulsar broker {}. "
- + "Check your broker configuration.",
- committable,
- ex);
- request.signalFailedWithKnownReason(ex);
- } else if (ex instanceof InvalidTxnStatusException) {
- LOG.error(
- "Unable to commit transaction ({}) because it's in an invalid state. "
- + "Most likely the transaction has been aborted for some reason. "
- + "Please check the Pulsar broker logs for more details.",
- committable,
- ex);
- request.signalAlreadyCommitted();
- } else if (ex instanceof TransactionNotFoundException) {
- if (request.getNumberOfRetries() == 0) {
- LOG.error(
- "Unable to commit transaction ({}) because it's not found on Pulsar broker. "
- + "Most likely the checkpoint interval exceed the transaction timeout.",
- committable,
- ex);
- request.signalFailedWithKnownReason(ex);
- } else {
- LOG.warn(
- "We can't find the transaction {} after {} retry committing. "
- + "This may mean that the transaction have been committed in previous but failed with timeout. "
- + "So we just mark it as committed.",
- txnID,
- request.getNumberOfRetries());
- request.signalAlreadyCommitted();
- }
- } else if (ex instanceof MetaStoreHandlerNotExistsException) {
- LOG.error(
- "We can't find the meta store handler by the mostSigBits from TxnID {}. "
- + "Did you change the metadata for topic {}?",
- committable,
- TRANSACTION_COORDINATOR_ASSIGN,
- ex);
- request.signalFailedWithKnownReason(ex);
- } else {
- LOG.error(
- "Encountered retriable exception while committing transaction {} for topic {}.",
- committable,
- topic,
- ex);
- int maxRecommitTimes = sinkConfiguration.getMaxRecommitTimes();
- if (request.getNumberOfRetries() < maxRecommitTimes) {
- request.retryLater();
- } else {
- String message =
- String.format(
- "Failed to commit transaction %s after retrying %d times",
- txnID, maxRecommitTimes);
- request.signalFailedWithKnownReason(new FlinkRuntimeException(message, ex));
- }
- }
- } catch (Exception e) {
- LOG.error(
- "Transaction ({}) encountered unknown error and data could be potentially lost.",
- committable,
- e);
- request.signalFailedWithUnknownReason(e);
- }
- }
- }
-
- /**
- * Lazy initialize this backend Pulsar client. This committer may not be used in {@link
- * DeliveryGuarantee#NONE} and {@link DeliveryGuarantee#AT_LEAST_ONCE}. So we couldn't create
- * the Pulsar client at first.
- */
- private TransactionCoordinatorClient transactionCoordinatorClient() {
- if (coordinatorClient == null) {
- this.pulsarClient = createClient(sinkConfiguration);
- this.coordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient();
-
- // Ensure you have enabled transaction.
- checkNotNull(coordinatorClient, "You haven't enable transaction in Pulsar client.");
- }
-
- return coordinatorClient;
- }
-
- @Override
- public void close() throws IOException {
- if (pulsarClient != null) {
- pulsarClient.close();
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/PulsarSinkConfigUtils.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/PulsarSinkConfigUtils.java
deleted file mode 100644
index 61cfd5af1e8..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/PulsarSinkConfigUtils.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.config;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.pulsar.common.config.PulsarConfigValidator;
-
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-
-import java.util.Map;
-import java.util.UUID;
-
-import static java.util.concurrent.TimeUnit.MICROSECONDS;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAMS;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAM_MAP;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_MEMORY_LIMIT_BYTES;
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_ENABLED;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_MAX_BYTES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_MAX_MESSAGES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_MAX_PUBLISH_DELAY_MICROS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_PARTITION_SWITCH_FREQUENCY_BY_PUBLISH_DELAY;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_CHUNKING_ENABLED;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_COMPRESSION_TYPE;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_INITIAL_SEQUENCE_ID;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_MAX_PENDING_MESSAGES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_PRODUCER_NAME;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_PRODUCER_PROPERTIES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_SEND_TIMEOUT_MS;
-import static org.apache.pulsar.client.api.MessageRoutingMode.SinglePartition;
-import static org.apache.pulsar.client.api.ProducerAccessMode.Shared;
-
-/** Create the {@link Producer} to send message and a validator for building sink config. */
-@Internal
-public final class PulsarSinkConfigUtils {
-
- private PulsarSinkConfigUtils() {
- // No need to create instance.
- }
-
- public static final PulsarConfigValidator SINK_CONFIG_VALIDATOR =
- PulsarConfigValidator.builder()
- .requiredOption(PULSAR_SERVICE_URL)
- .requiredOption(PULSAR_ADMIN_URL)
- .conflictOptions(PULSAR_AUTH_PARAMS, PULSAR_AUTH_PARAM_MAP)
- .conflictOptions(PULSAR_MEMORY_LIMIT_BYTES, PULSAR_MAX_PENDING_MESSAGES)
- .conflictOptions(
- PULSAR_MEMORY_LIMIT_BYTES,
- PULSAR_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)
- .build();
-
- /** Create a pulsar producer builder by using the given Configuration. */
- public static <T> ProducerBuilder<T> createProducerBuilder(
- PulsarClient client, Schema<T> schema, SinkConfiguration configuration) {
- ProducerBuilder<T> builder = client.newProducer(schema);
-
- configuration.useOption(
- PULSAR_PRODUCER_NAME,
- producerName -> String.format(producerName, UUID.randomUUID()),
- builder::producerName);
- configuration.useOption(
- PULSAR_SEND_TIMEOUT_MS,
- Math::toIntExact,
- ms -> builder.sendTimeout(ms, MILLISECONDS));
- configuration.useOption(
- PULSAR_BATCHING_MAX_PUBLISH_DELAY_MICROS,
- s -> builder.batchingMaxPublishDelay(s, MICROSECONDS));
- configuration.useOption(
- PULSAR_BATCHING_PARTITION_SWITCH_FREQUENCY_BY_PUBLISH_DELAY,
- builder::roundRobinRouterBatchingPartitionSwitchFrequency);
- configuration.useOption(PULSAR_BATCHING_MAX_MESSAGES, builder::batchingMaxMessages);
- configuration.useOption(PULSAR_BATCHING_MAX_BYTES, builder::batchingMaxBytes);
- configuration.useOption(PULSAR_BATCHING_ENABLED, builder::enableBatching);
- configuration.useOption(PULSAR_CHUNKING_ENABLED, builder::enableChunking);
- configuration.useOption(PULSAR_COMPRESSION_TYPE, builder::compressionType);
- configuration.useOption(PULSAR_INITIAL_SEQUENCE_ID, builder::initialSequenceId);
-
- // Set producer properties
- Map<String, String> properties = configuration.getProperties(PULSAR_PRODUCER_PROPERTIES);
- if (!properties.isEmpty()) {
- builder.properties(properties);
- }
-
- // Set the default value for current producer builder.
- // We use non-partitioned producer by default. This wouldn't be changed in the future.
- builder.blockIfQueueFull(true)
- .messageRoutingMode(SinglePartition)
- .enableMultiSchema(false)
- .autoUpdatePartitions(false)
- .accessMode(Shared)
- .enableLazyStartPartitionedProducers(false);
-
- return builder;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/SinkConfiguration.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/SinkConfiguration.java
deleted file mode 100644
index 0a56ff34fa0..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/config/SinkConfiguration.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.config;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.connector.sink.Sink.InitContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.common.config.PulsarConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.PulsarWriter;
-import org.apache.flink.connector.pulsar.sink.writer.router.MessageKeyHash;
-import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSchemaWrapper;
-
-import org.apache.pulsar.client.api.Schema;
-
-import java.util.Objects;
-
-import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_MAX_MESSAGES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_ENABLE_SINK_METRICS;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_MAX_RECOMMIT_TIMES;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_MESSAGE_KEY_HASH;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_TOPIC_METADATA_REFRESH_INTERVAL;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_DELIVERY_GUARANTEE;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_SCHEMA_EVOLUTION;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_TRANSACTION_TIMEOUT;
-
-/** The configured class for pulsar sink. */
-@PublicEvolving
-public class SinkConfiguration extends PulsarConfiguration {
- private static final long serialVersionUID = 4941360605051251153L;
-
- private final DeliveryGuarantee deliveryGuarantee;
- private final long transactionTimeoutMillis;
- private final long topicMetadataRefreshInterval;
- private final int partitionSwitchSize;
- private final MessageKeyHash messageKeyHash;
- private final boolean enableSchemaEvolution;
- private final int maxRecommitTimes;
- private final boolean enableMetrics;
-
- public SinkConfiguration(Configuration configuration) {
- super(configuration);
-
- this.deliveryGuarantee = get(PULSAR_WRITE_DELIVERY_GUARANTEE);
- this.transactionTimeoutMillis = getLong(PULSAR_WRITE_TRANSACTION_TIMEOUT);
- this.topicMetadataRefreshInterval = getLong(PULSAR_TOPIC_METADATA_REFRESH_INTERVAL);
- this.partitionSwitchSize = getInteger(PULSAR_BATCHING_MAX_MESSAGES);
- this.messageKeyHash = get(PULSAR_MESSAGE_KEY_HASH);
- this.enableSchemaEvolution = get(PULSAR_WRITE_SCHEMA_EVOLUTION);
- this.maxRecommitTimes = get(PULSAR_MAX_RECOMMIT_TIMES);
- this.enableMetrics =
- get(PULSAR_ENABLE_SINK_METRICS) && get(PULSAR_STATS_INTERVAL_SECONDS) > 0;
- }
-
- /** The delivery guarantee changes the behavior of {@link PulsarWriter}. */
- public DeliveryGuarantee getDeliveryGuarantee() {
- return deliveryGuarantee;
- }
-
- /**
- * Pulsar's transactions have a timeout mechanism for the uncommitted transaction. We use
- * transactions for making sure the message could be written only once. Since the checkpoint
- * interval couldn't be acquired from {@link InitContext}, we have to expose this option. Make
- * sure this value is greater than the checkpoint interval. Create a pulsar producer builder by
- * using the given Configuration.
- */
- public long getTransactionTimeoutMillis() {
- return transactionTimeoutMillis;
- }
-
- /**
- * Auto-update the topic metadata in a fixed interval (in ms). The default value is 30 minutes.
- */
- public long getTopicMetadataRefreshInterval() {
- return topicMetadataRefreshInterval;
- }
-
- /**
- * Switch the partition to write when we have written the given size of messages. It's used for
- * a round-robin topic router.
- */
- public int getPartitionSwitchSize() {
- return partitionSwitchSize;
- }
-
- /** The message key's hash logic for routing the message into one Pulsar partition. */
- public MessageKeyHash getMessageKeyHash() {
- return messageKeyHash;
- }
-
- /**
- * If we should serialize and send the message with a specified Pulsar {@link Schema} instead
- * the default {@link Schema#BYTES}. This switch is only used for {@link PulsarSchemaWrapper}.
- */
- public boolean isEnableSchemaEvolution() {
- return enableSchemaEvolution;
- }
-
- /** The maximum allowed recommitting time for a Pulsar transaction. */
- public int getMaxRecommitTimes() {
- return maxRecommitTimes;
- }
-
- /** Whether to expose the metrics from Pulsar Producer. */
- public boolean isEnableMetrics() {
- return enableMetrics;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- SinkConfiguration that = (SinkConfiguration) o;
- return transactionTimeoutMillis == that.transactionTimeoutMillis
- && topicMetadataRefreshInterval == that.topicMetadataRefreshInterval
- && partitionSwitchSize == that.partitionSwitchSize
- && enableSchemaEvolution == that.enableSchemaEvolution
- && messageKeyHash == that.messageKeyHash
- && maxRecommitTimes == that.maxRecommitTimes
- && enableMetrics == that.enableMetrics;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(
- super.hashCode(),
- transactionTimeoutMillis,
- topicMetadataRefreshInterval,
- partitionSwitchSize,
- messageKeyHash,
- enableSchemaEvolution,
- maxRecommitTimes,
- enableMetrics);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
deleted file mode 100644
index c5c4622a998..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.operators.MailboxExecutor;
-import org.apache.flink.api.common.operators.ProcessingTimeService;
-import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext;
-import org.apache.flink.api.connector.sink2.Sink.InitContext;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink.PrecommittingSinkWriter;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContextImpl;
-import org.apache.flink.connector.pulsar.sink.writer.delayer.MessageDelayer;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessage;
-import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
-import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
-import org.apache.flink.connector.pulsar.sink.writer.topic.TopicMetadataListener;
-import org.apache.flink.connector.pulsar.sink.writer.topic.TopicProducerRegister;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.flink.shaded.guava30.com.google.common.base.Strings;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static java.util.Collections.emptyList;
-import static org.apache.flink.util.IOUtils.closeAll;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * This class is responsible to write records in a Pulsar topic and to handle the different delivery
- * {@link DeliveryGuarantee}s.
- *
- * @param <IN> The type of the input elements.
- */
-@Internal
-public class PulsarWriter<IN> implements PrecommittingSinkWriter<IN, PulsarCommittable> {
- private static final Logger LOG = LoggerFactory.getLogger(PulsarWriter.class);
-
- private final PulsarSerializationSchema<IN> serializationSchema;
- private final TopicMetadataListener metadataListener;
- private final TopicRouter<IN> topicRouter;
- private final MessageDelayer<IN> messageDelayer;
- private final DeliveryGuarantee deliveryGuarantee;
- private final PulsarSinkContext sinkContext;
- private final TopicProducerRegister producerRegister;
- private final MailboxExecutor mailboxExecutor;
- private final AtomicLong pendingMessages;
-
- /**
- * Constructor creating a Pulsar writer.
- *
- * <p>It will throw a {@link RuntimeException} if {@link
- * PulsarSerializationSchema#open(InitializationContext, PulsarSinkContext, SinkConfiguration)}
- * fails.
- *
- * @param sinkConfiguration The configuration to configure the Pulsar producer.
- * @param serializationSchema Transform the incoming records into different message properties.
- * @param metadataListener The listener for querying topic metadata.
- * @param topicRouter Topic router to choose topic by incoming records.
- * @param initContext Context to provide information about the runtime environment.
- */
- public PulsarWriter(
- SinkConfiguration sinkConfiguration,
- PulsarSerializationSchema<IN> serializationSchema,
- TopicMetadataListener metadataListener,
- TopicRouter<IN> topicRouter,
- MessageDelayer<IN> messageDelayer,
- InitContext initContext) {
- checkNotNull(sinkConfiguration);
- this.serializationSchema = checkNotNull(serializationSchema);
- this.metadataListener = checkNotNull(metadataListener);
- this.topicRouter = checkNotNull(topicRouter);
- this.messageDelayer = checkNotNull(messageDelayer);
- checkNotNull(initContext);
-
- this.deliveryGuarantee = sinkConfiguration.getDeliveryGuarantee();
- this.sinkContext = new PulsarSinkContextImpl(initContext, sinkConfiguration);
-
- // Initialize topic metadata listener.
- LOG.debug("Initialize topic metadata after creating Pulsar writer.");
- ProcessingTimeService timeService = initContext.getProcessingTimeService();
- this.metadataListener.open(sinkConfiguration, timeService);
-
- // Initialize topic router.
- this.topicRouter.open(sinkConfiguration);
-
- // Initialize the serialization schema.
- try {
- InitializationContext initializationContext =
- initContext.asSerializationSchemaInitializationContext();
- this.serializationSchema.open(initializationContext, sinkContext, sinkConfiguration);
- } catch (Exception e) {
- throw new FlinkRuntimeException("Cannot initialize schema.", e);
- }
-
- // Create this producer register after opening serialization schema!
- this.producerRegister =
- new TopicProducerRegister(sinkConfiguration, initContext.metricGroup());
- this.mailboxExecutor = initContext.getMailboxExecutor();
- this.pendingMessages = new AtomicLong(0);
- }
-
- @Override
- public void write(IN element, Context context) throws IOException, InterruptedException {
- PulsarMessage<?> message = serializationSchema.serialize(element, sinkContext);
-
- // Choose the right topic to send.
- String key = message.getKey();
- List<String> availableTopics = metadataListener.availableTopics();
- String topic = topicRouter.route(element, key, availableTopics, sinkContext);
-
- // Create message builder for sending message.
- TypedMessageBuilder<?> builder = createMessageBuilder(topic, context, message);
-
- // Message Delay delivery.
- long deliverAt = messageDelayer.deliverAt(element, sinkContext);
- if (deliverAt > 0) {
- builder.deliverAt(deliverAt);
- }
-
- // Perform message sending.
- if (deliveryGuarantee == DeliveryGuarantee.NONE) {
- // We would just ignore the sending exception. This may cause data loss.
- builder.sendAsync();
- } else {
- // Increase the pending message count.
- pendingMessages.incrementAndGet();
- builder.sendAsync()
- .whenComplete(
- (id, ex) -> {
- pendingMessages.decrementAndGet();
- if (ex != null) {
- mailboxExecutor.execute(
- () -> {
- throw new FlinkRuntimeException(
- "Failed to send data to Pulsar " + topic,
- ex);
- },
- "Failed to send data to Pulsar");
- } else {
- LOG.debug(
- "Sent message to Pulsar {} with message id {}",
- topic,
- id);
- }
- });
- }
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private TypedMessageBuilder<?> createMessageBuilder(
- String topic, Context context, PulsarMessage<?> message) {
-
- Schema<?> schema = message.getSchema();
- TypedMessageBuilder<?> builder = producerRegister.createMessageBuilder(topic, schema);
-
- byte[] orderingKey = message.getOrderingKey();
- if (orderingKey != null && orderingKey.length > 0) {
- builder.orderingKey(orderingKey);
- }
-
- String key = message.getKey();
- if (!Strings.isNullOrEmpty(key)) {
- builder.key(key);
- }
-
- long eventTime = message.getEventTime();
- if (eventTime > 0) {
- builder.eventTime(eventTime);
- } else {
- // Set default message timestamp if flink has provided one.
- Long timestamp = context.timestamp();
- if (timestamp != null && timestamp > 0L) {
- builder.eventTime(timestamp);
- }
- }
-
- // Schema evolution would serialize the message by Pulsar Schema in TypedMessageBuilder.
- // The type has been checked in PulsarMessageBuilder#value.
- ((TypedMessageBuilder) builder).value(message.getValue());
-
- Map<String, String> properties = message.getProperties();
- if (properties != null && !properties.isEmpty()) {
- builder.properties(properties);
- }
-
- Long sequenceId = message.getSequenceId();
- if (sequenceId != null) {
- builder.sequenceId(sequenceId);
- }
-
- List<String> clusters = message.getReplicationClusters();
- if (clusters != null && !clusters.isEmpty()) {
- builder.replicationClusters(clusters);
- }
-
- if (message.isDisableReplication()) {
- builder.disableReplication();
- }
-
- return builder;
- }
-
- @Override
- public void flush(boolean endOfInput) throws IOException {
- if (endOfInput || deliveryGuarantee != DeliveryGuarantee.NONE) {
- LOG.info("Flush the pending messages to Pulsar.");
-
- // Try to flush pending messages.
- producerRegister.flush();
- // Make sure all the pending messages should be flushed to Pulsar.
- while (pendingMessages.longValue() > 0) {
- producerRegister.flush();
- }
- }
- }
-
- @Override
- public Collection<PulsarCommittable> prepareCommit() {
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- return producerRegister.prepareCommit();
- } else {
- return emptyList();
- }
- }
-
- @Override
- public void close() throws Exception {
- // Close all the resources and throw the exception at last.
- closeAll(metadataListener, producerRegister);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContext.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContext.java
deleted file mode 100644
index 5c9333952c0..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContext.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.context;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-/** This context provides information on the pulsar record target location. */
-@PublicEvolving
-public interface PulsarSinkContext {
-
- /**
- * Get the number of the subtask that PulsarSink is running on. The numbering starts from 0 and
- * goes up to parallelism-1. (parallelism as returned by {@link #getNumberOfParallelInstances()}
- *
- * @return number of subtask
- */
- int getParallelInstanceId();
-
- /** @return number of parallel PulsarSink tasks. */
- int getNumberOfParallelInstances();
-
- /**
- * Pulsar can check the schema and upgrade the schema automatically. If you enable this option,
- * we wouldn't serialize the record into bytes, we send and serialize it in the client.
- */
- boolean isEnableSchemaEvolution();
-
- /** Returns the current process time in flink. */
- long processTime();
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContextImpl.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContextImpl.java
deleted file mode 100644
index 681b25a0569..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/context/PulsarSinkContextImpl.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.context;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.operators.ProcessingTimeService;
-import org.apache.flink.api.connector.sink2.Sink.InitContext;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-
-/** An implementation that would contain all the required context. */
-@Internal
-public class PulsarSinkContextImpl implements PulsarSinkContext {
-
- private final int numberOfParallelSubtasks;
- private final int parallelInstanceId;
- private final ProcessingTimeService processingTimeService;
- private final boolean enableSchemaEvolution;
-
- public PulsarSinkContextImpl(InitContext initContext, SinkConfiguration sinkConfiguration) {
- this.parallelInstanceId = initContext.getSubtaskId();
- this.numberOfParallelSubtasks = initContext.getNumberOfParallelSubtasks();
- this.processingTimeService = initContext.getProcessingTimeService();
- this.enableSchemaEvolution = sinkConfiguration.isEnableSchemaEvolution();
- }
-
- @Override
- public int getParallelInstanceId() {
- return parallelInstanceId;
- }
-
- @Override
- public int getNumberOfParallelInstances() {
- return numberOfParallelSubtasks;
- }
-
- @Override
- public boolean isEnableSchemaEvolution() {
- return enableSchemaEvolution;
- }
-
- @Override
- public long processTime() {
- return processingTimeService.getCurrentProcessingTime();
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/FixedMessageDelayer.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/FixedMessageDelayer.java
deleted file mode 100644
index c11d2f8ad49..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/FixedMessageDelayer.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.delayer;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-
-/** A delayer for making sure all the messages would be sent in a fixed delay duration. */
-@PublicEvolving
-public class FixedMessageDelayer<IN> implements MessageDelayer<IN> {
- private static final long serialVersionUID = -7550834520312097614L;
-
- private final long delayDuration;
-
- public FixedMessageDelayer(long delayDuration) {
- this.delayDuration = delayDuration;
- }
-
- @Override
- public long deliverAt(IN message, PulsarSinkContext sinkContext) {
- if (delayDuration > 0) {
- return sinkContext.processTime() + delayDuration;
- } else {
- return delayDuration;
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/MessageDelayer.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/MessageDelayer.java
deleted file mode 100644
index 53a345b0466..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/delayer/MessageDelayer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.delayer;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-
-import org.apache.pulsar.client.api.SubscriptionType;
-
-import java.io.Serializable;
-import java.time.Duration;
-
-/**
- * A delayer for Pulsar broker passing the sent message to the downstream consumer. This is only
- * works in {@link SubscriptionType#Shared} subscription.
- *
- * <p>Read <a
- * href="https://pulsar.apache.org/docs/en/next/concepts-messaging/#delayed-message-delivery">delayed
- * message delivery</a> for better understanding this feature.
- */
-@PublicEvolving
-public interface MessageDelayer<IN> extends Serializable {
-
- /**
- * Return the send time for this message. You should calculate the timestamp by using {@link
- * PulsarSinkContext#processTime()} and the non-positive value indicate this message should be
- * sent immediately.
- */
- long deliverAt(IN message, PulsarSinkContext sinkContext);
-
- /** Implement this method if you have some non-serializable field. */
- default void open(SinkConfiguration sinkConfiguration) {
- // Nothing to do by default.
- }
-
- /** All the messages should be consumed immediately. */
- static <IN> FixedMessageDelayer<IN> never() {
- return new FixedMessageDelayer<>(-1L);
- }
-
- /** All the messages should be consumed in a fixed duration. */
- static <IN> FixedMessageDelayer<IN> fixed(Duration duration) {
- return new FixedMessageDelayer<>(duration.toMillis());
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
deleted file mode 100644
index 0c45763cdb1..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.message;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-
-import javax.annotation.Nullable;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * The message instance would be used for {@link TypedMessageBuilder}. We create this class because
- * the Pulsar lacks such kind of POJO class.
- */
-@PublicEvolving
-public class PulsarMessage<T> {
-
- @Nullable private final byte[] orderingKey;
- @Nullable private final String key;
- private final long eventTime;
- private final Schema<T> schema;
- @Nullable private final T value;
- @Nullable private final Map<String, String> properties;
- @Nullable private final Long sequenceId;
- @Nullable private final List<String> replicationClusters;
- private final boolean disableReplication;
-
- /** Package private for building this class only in {@link PulsarMessageBuilder}. */
- PulsarMessage(
- @Nullable byte[] orderingKey,
- @Nullable String key,
- long eventTime,
- Schema<T> schema,
- @Nullable T value,
- @Nullable Map<String, String> properties,
- @Nullable Long sequenceId,
- @Nullable List<String> replicationClusters,
- boolean disableReplication) {
- this.orderingKey = orderingKey;
- this.key = key;
- this.eventTime = eventTime;
- this.schema = schema;
- this.value = value;
- this.properties = properties;
- this.sequenceId = sequenceId;
- this.replicationClusters = replicationClusters;
- this.disableReplication = disableReplication;
- }
-
- @Nullable
- public byte[] getOrderingKey() {
- return orderingKey;
- }
-
- @Nullable
- public String getKey() {
- return key;
- }
-
- public long getEventTime() {
- return eventTime;
- }
-
- public Schema<T> getSchema() {
- return schema;
- }
-
- @Nullable
- public T getValue() {
- return value;
- }
-
- @Nullable
- public Map<String, String> getProperties() {
- return properties;
- }
-
- @Nullable
- public Long getSequenceId() {
- return sequenceId;
- }
-
- @Nullable
- public List<String> getReplicationClusters() {
- return replicationClusters;
- }
-
- public boolean isDisableReplication() {
- return disableReplication;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
deleted file mode 100644
index 518593282ef..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.message;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.pulsar.sink.writer.router.KeyHashTopicRouter;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/** {@link TypedMessageBuilder} wrapper for providing the required method for end-users. */
-@PublicEvolving
-public class PulsarMessageBuilder<T> {
-
- private byte[] orderingKey;
- private String key;
- private long eventTime;
- Schema<T> schema;
- private T value;
- private Map<String, String> properties = new HashMap<>();
- private Long sequenceId;
- private List<String> replicationClusters;
- private boolean disableReplication = false;
-
- /** Method wrapper of {@link TypedMessageBuilder#orderingKey(byte[])}. */
- public PulsarMessageBuilder<T> orderingKey(byte[] orderingKey) {
- this.orderingKey = checkNotNull(orderingKey);
- return this;
- }
-
- /**
- * Property {@link TypedMessageBuilder#key(String)}. This property would also be used in {@link
- * KeyHashTopicRouter}.
- */
- public PulsarMessageBuilder<T> key(String key) {
- this.key = checkNotNull(key);
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#eventTime(long)}. */
- public PulsarMessageBuilder<T> eventTime(long eventTime) {
- this.eventTime = eventTime;
- return this;
- }
-
- /**
- * Method wrapper of {@link TypedMessageBuilder#value(Object)}. You can pass any schema for
- * validating it on Pulsar. This is called schema evolution. But the topic on Pulsar should bind
- * to a fixed {@link Schema}. You can't have multiple schemas on the same topic unless it's
- * compatible with each other.
- *
- * @param value The value could be null, which is called tombstones message in Pulsar. (It will
- * be skipped and considered deleted.)
- */
- public PulsarMessageBuilder<T> value(Schema<T> schema, T value) {
- this.schema = checkNotNull(schema);
- this.value = value;
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#property(String, String)}. */
- public PulsarMessageBuilder<T> property(String key, String value) {
- this.properties.put(checkNotNull(key), checkNotNull(value));
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#properties(Map)}. */
- public PulsarMessageBuilder<T> properties(Map<String, String> properties) {
- this.properties.putAll(checkNotNull(properties));
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#sequenceId(long)}. */
- public PulsarMessageBuilder<T> sequenceId(long sequenceId) {
- this.sequenceId = sequenceId;
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#replicationClusters(List)}. */
- public PulsarMessageBuilder<T> replicationClusters(List<String> replicationClusters) {
- this.replicationClusters = checkNotNull(replicationClusters);
- return this;
- }
-
- /** Method wrapper of {@link TypedMessageBuilder#disableReplication()}. */
- public PulsarMessageBuilder<T> disableReplication() {
- this.disableReplication = true;
- return this;
- }
-
- public PulsarMessage<T> build() {
- checkNotNull(schema, "Schema should be provided.");
-
- return new PulsarMessage<>(
- orderingKey,
- key,
- eventTime,
- schema,
- value,
- properties,
- sequenceId,
- replicationClusters,
- disableReplication);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/KeyHashTopicRouter.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/KeyHashTopicRouter.java
deleted file mode 100644
index 433d79c6cbc..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/KeyHashTopicRouter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.router;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-
-import org.apache.flink.shaded.guava30.com.google.common.base.Strings;
-
-import org.apache.pulsar.client.impl.Hash;
-
-import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
-
-import static org.apache.flink.shaded.guava30.com.google.common.base.Preconditions.checkArgument;
-import static org.apache.pulsar.client.util.MathUtils.signSafeMod;
-
-/**
- * If you choose the {@link TopicRoutingMode#MESSAGE_KEY_HASH} policy, we would use this
- * implementation. We would pick the topic by the message key's hash code. If no message key was
- * provided, we would randomly pick one.
- *
- * @param <IN> The message type which should write to Pulsar.
- */
-@Internal
-public class KeyHashTopicRouter<IN> implements TopicRouter<IN> {
- private static final long serialVersionUID = 2475614648095079804L;
-
- private final MessageKeyHash messageKeyHash;
-
- public KeyHashTopicRouter(SinkConfiguration sinkConfiguration) {
- this.messageKeyHash = sinkConfiguration.getMessageKeyHash();
- }
-
- @Override
- public String route(IN in, String key, List<String> partitions, PulsarSinkContext context) {
- checkArgument(
- !partitions.isEmpty(),
- "You should provide topics for routing topic by message key hash.");
-
- int topicIndex;
- if (Strings.isNullOrEmpty(key)) {
- // We would randomly pick one topic to write.
- topicIndex = ThreadLocalRandom.current().nextInt(partitions.size());
- } else {
- // Hash the message key and choose the topic to write.
- Hash hash = messageKeyHash.getHash();
- int code = hash.makeHash(key);
- topicIndex = signSafeMod(code, partitions.size());
- }
-
- return partitions.get(topicIndex);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/MessageKeyHash.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/MessageKeyHash.java
deleted file mode 100644
index 3ad092d848a..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/MessageKeyHash.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.router;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.configuration.DescribedEnum;
-import org.apache.flink.configuration.description.InlineElement;
-
-import org.apache.pulsar.client.impl.Hash;
-import org.apache.pulsar.client.impl.JavaStringHash;
-import org.apache.pulsar.client.impl.Murmur3Hash32;
-
-import static org.apache.flink.configuration.description.LinkElement.link;
-import static org.apache.flink.configuration.description.TextElement.code;
-import static org.apache.flink.configuration.description.TextElement.text;
-
-/** Predefined the available hash function for routing the message. */
-@PublicEvolving
-public enum MessageKeyHash implements DescribedEnum {
-
- /** Use regular <code>String.hashCode()</code>. */
- JAVA_HASH(
- "java-hash",
- text(
- "This hash would use %s to calculate the message key string's hash code.",
- code("String.hashCode()"))) {
- @Override
- public Hash getHash() {
- return JavaStringHash.getInstance();
- }
- },
- /**
- * Use Murmur3 hashing function. <a
- * href="https://en.wikipedia.org/wiki/MurmurHash">https://en.wikipedia.org/wiki/MurmurHash</a>
- */
- MURMUR3_32_HASH(
- "murmur-3-32-hash",
- text(
- "This hash would calculate message key's hash code by using %s algorithm.",
- link("https://en.wikipedia.org/wiki/MurmurHash", "Murmur3"))) {
- @Override
- public Hash getHash() {
- return Murmur3Hash32.getInstance();
- }
- };
-
- private final String name;
- private final transient InlineElement desc;
-
- MessageKeyHash(String name, InlineElement desc) {
- this.name = name;
- this.desc = desc;
- }
-
- @Internal
- public abstract Hash getHash();
-
- @Override
- public String toString() {
- return name;
- }
-
- @Internal
- @Override
- public InlineElement getDescription() {
- return desc;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouter.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouter.java
deleted file mode 100644
index 65e51548951..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/RoundRobinTopicRouter.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.router;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.util.Preconditions;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.flink.shaded.guava30.com.google.common.base.Preconditions.checkArgument;
-
-/**
- * If you choose the {@link TopicRoutingMode#ROUND_ROBIN} policy, we would use this implementation.
- * We would pick the topic one by one in a fixed batch size.
- *
- * @param <IN> The message type which should write to Pulsar.
- */
-@Internal
-public class RoundRobinTopicRouter<IN> implements TopicRouter<IN> {
- private static final long serialVersionUID = -1160533263474038206L;
-
- /** The internal counter for counting the messages. */
- private final AtomicLong counter = new AtomicLong(0);
-
- /** The size when we switch to another topic. */
- private final int partitionSwitchSize;
-
- public RoundRobinTopicRouter(SinkConfiguration configuration) {
- this.partitionSwitchSize = configuration.getPartitionSwitchSize();
- Preconditions.checkArgument(partitionSwitchSize > 0);
- }
-
- @Override
- public String route(IN in, String key, List<String> partitions, PulsarSinkContext context) {
- checkArgument(
- !partitions.isEmpty(),
- "You should provide topics for routing topic by message key hash.");
-
- long counts = counter.getAndAdd(1);
- long index = (counts / partitionSwitchSize) % partitions.size();
- // Avoid digit overflow for message counter.
- int topicIndex = (int) (Math.abs(index) % Integer.MAX_VALUE);
-
- return partitions.get(topicIndex);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRouter.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRouter.java
deleted file mode 100644
index a2c05892116..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRouter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.router;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.pulsar.sink.PulsarSinkBuilder;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
-import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * The router for choosing the desired topic to write the Flink records. The user can implement this
- * router for complex requirements. We have provided some easy-to-use implementations.
- *
- * <p>This topic router is stateless and doesn't have any initialize logic. Make sure you don't
- * require some dynamic state.
- *
- * @param <IN> The record type needs to be written to Pulsar.
- */
-@PublicEvolving
-public interface TopicRouter<IN> extends Serializable {
-
- /**
- * Choose the topic by given record & available partition list. You can return a new topic name
- * if you need it.
- *
- * @param in The record instance which need to be written to Pulsar.
- * @param key The key of the message from {@link PulsarMessageBuilder#key(String)}. It could be
- * null, if message doesn't have a key.
- * @param partitions The available partition list. This could be empty if you don't provide any
- * topics in {@link PulsarSinkBuilder#setTopics(String...)}. You can return a custom topic,
- * but make sure it should contain a partition index in naming. Using {@link
- * TopicNameUtils#topicNameWithPartition(String, int)} can easily create a topic name with
- * partition index.
- * @param context The context contains useful information for determining the topic.
- * @return The topic name to use.
- */
- String route(IN in, String key, List<String> partitions, PulsarSinkContext context);
-
- /** Implement this method if you have some non-serializable field. */
- default void open(SinkConfiguration sinkConfiguration) {
- // Nothing to do by default.
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRoutingMode.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRoutingMode.java
deleted file mode 100644
index f251f8f760f..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/router/TopicRoutingMode.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.router;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.configuration.DescribedEnum;
-import org.apache.flink.configuration.description.InlineElement;
-
-import static org.apache.flink.configuration.description.TextElement.code;
-import static org.apache.flink.configuration.description.TextElement.text;
-import static org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_BATCHING_MAX_MESSAGES;
-
-/** The routing policy for choosing the desired topic by the given message. */
-@PublicEvolving
-public enum TopicRoutingMode implements DescribedEnum {
-
- /**
- * The producer will publish messages across all partitions in a round-robin fashion to achieve
- * maximum throughput. Please note that round-robin is not done per individual message but
- * rather it's set to the same boundary of batching delay, to ensure batching is effective.
- */
- ROUND_ROBIN(
- "round-robin",
- text(
- "The producer will publish messages across all partitions in a round-robin fashion to achieve maximum throughput."
- + " Please note that round-robin is not done per individual message"
- + " but rather it's set to the same boundary of %s, to ensure batching is effective.",
- code(PULSAR_BATCHING_MAX_MESSAGES.key()))),
-
- /**
- * If no key is provided, The partitioned producer will randomly pick one single topic partition
- * and publish all the messages into that partition. If a key is provided on the message, the
- * partitioned producer will hash the key and assign the message to a particular partition.
- */
- MESSAGE_KEY_HASH(
- "message-key-hash",
- text(
- "If no key is provided, The partitioned producer will randomly pick one single topic partition"
- + " and publish all the messages into that partition. If a key is provided on the message,"
- + " the partitioned producer will hash the key and assign the message to a particular partition.")),
-
- /**
- * Use custom topic router implementation that will be called to determine the partition for a
- * particular message.
- */
- CUSTOM(
- "custom",
- text(
- "Use custom %s implementation that will be called to determine the partition for a particular message.",
- code(TopicRouter.class.getSimpleName())));
-
- private final String name;
- private final transient InlineElement desc;
-
- TopicRoutingMode(String name, InlineElement desc) {
- this.name = name;
- this.desc = desc;
- }
-
- @Internal
- @Override
- public InlineElement getDescription() {
- return desc;
- }
-
- @Override
- public String toString() {
- return name;
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSchemaWrapper.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSchemaWrapper.java
deleted file mode 100644
index 0d5aaf033c5..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSchemaWrapper.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.serializer;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchema;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessage;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
-
-import org.apache.pulsar.client.api.Schema;
-
-/**
- * Wrap the Pulsar's Schema into PulsarSerializationSchema. We support schema evolution out of box
- * by this implementation.
- */
-@Internal
-public class PulsarSchemaWrapper<IN> implements PulsarSerializationSchema<IN> {
- private static final long serialVersionUID = -2567052498398184194L;
-
- private final PulsarSchema<IN> pulsarSchema;
-
- public PulsarSchemaWrapper(PulsarSchema<IN> pulsarSchema) {
- this.pulsarSchema = pulsarSchema;
- }
-
- @Override
- public PulsarMessage<?> serialize(IN element, PulsarSinkContext sinkContext) {
- Schema<IN> schema = this.pulsarSchema.getPulsarSchema();
- if (sinkContext.isEnableSchemaEvolution()) {
- PulsarMessageBuilder<IN> builder = new PulsarMessageBuilder<>();
- builder.value(schema, element);
-
- return builder.build();
- } else {
- PulsarMessageBuilder<byte[]> builder = new PulsarMessageBuilder<>();
- byte[] bytes = schema.encode(element);
- builder.value(Schema.BYTES, bytes);
-
- return builder.build();
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchema.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchema.java
deleted file mode 100644
index da7f70652b7..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchema.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.serializer;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext;
-import org.apache.flink.connector.pulsar.common.schema.PulsarSchema;
-import org.apache.flink.connector.pulsar.sink.PulsarSinkBuilder;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessage;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
-
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.common.schema.KeyValue;
-
-import java.io.Serializable;
-
-/**
- * The serialization schema for how to serialize records into Pulsar.
- *
- * @param <IN> The message type send to Pulsar.
- */
-@PublicEvolving
-public interface PulsarSerializationSchema<IN> extends Serializable {
-
- /**
- * Initialization method for the schema. It is called before the actual working methods {@link
- * #serialize(Object, PulsarSinkContext)} and thus suitable for one-time setup work.
- *
- * <p>The provided {@link InitializationContext} can be used to access additional features such
- * as registering user metrics.
- *
- * @param initializationContext Contextual information that can be used during initialization.
- * @param sinkContext Runtime information i.e. partitions, subtaskId.
- * @param sinkConfiguration All the configure options for the Pulsar sink. You can add custom
- * options.
- */
- default void open(
- InitializationContext initializationContext,
- PulsarSinkContext sinkContext,
- SinkConfiguration sinkConfiguration)
- throws Exception {
- // Nothing to do by default.
- }
-
- /**
- * Serializes the given element into bytes and {@link Schema#BYTES}. Or you can convert it to a
- * new type of instance with a {@link Schema}. The return value {@link PulsarMessage} can be
- * built by {@link PulsarMessageBuilder}. All the methods provided in the {@link
- * PulsarMessageBuilder} is just equals to the {@link TypedMessageBuilder}.
- *
- * @param element Element to be serialized.
- * @param sinkContext Context to provide extra information.
- */
- PulsarMessage<?> serialize(IN element, PulsarSinkContext sinkContext);
-
- /**
- * Create a PulsarSerializationSchema by using the flink's {@link SerializationSchema}. It would
- * serialize the message into byte array and send it to Pulsar with {@link Schema#BYTES}.
- */
- static <T> PulsarSerializationSchema<T> flinkSchema(
- SerializationSchema<T> serializationSchema) {
- return new PulsarSerializationSchemaWrapper<>(serializationSchema);
- }
-
- /**
- * Create a PulsarSerializationSchema by using the Pulsar {@link Schema} instance. We can send
- * message with the given schema to Pulsar, this would be enabled by {@link
- * PulsarSinkBuilder#enableSchemaEvolution()}. We would serialize the message into bytes and
- * send it as {@link Schema#BYTES} by default.
- *
- * <p>We only support <a
- * href="https://pulsar.apache.org/docs/en/schema-understand/#primitive-type">primitive
- * types</a> here.
- */
- static <T> PulsarSerializationSchema<T> pulsarSchema(Schema<T> schema) {
- PulsarSchema<T> pulsarSchema = new PulsarSchema<>(schema);
- return new PulsarSchemaWrapper<>(pulsarSchema);
- }
-
- /**
- * Create a PulsarSerializationSchema by using the Pulsar {@link Schema} instance. We can send
- * message with the given schema to Pulsar, this would be enabled by {@link
- * PulsarSinkBuilder#enableSchemaEvolution()}. We would serialize the message into bytes and
- * send it as {@link Schema#BYTES} by default.
- *
- * <p>We only support <a
- * href="https://pulsar.apache.org/docs/en/schema-understand/#struct">struct types</a> here.
- */
- static <T> PulsarSerializationSchema<T> pulsarSchema(Schema<T> schema, Class<T> typeClass) {
- PulsarSchema<T> pulsarSchema = new PulsarSchema<>(schema, typeClass);
- return new PulsarSchemaWrapper<>(pulsarSchema);
- }
-
- /**
- * Create a PulsarSerializationSchema by using the Pulsar {@link Schema} instance. We can send
- * message with the given schema to Pulsar, this would be enabled by {@link
- * PulsarSinkBuilder#enableSchemaEvolution()}. We would serialize the message into bytes and
- * send it as {@link Schema#BYTES} by default.
- *
- * <p>We only support <a
- * href="https://pulsar.apache.org/docs/en/schema-understand/#keyvalue">keyvalue types</a> here.
- */
- static <K, V> PulsarSerializationSchema<KeyValue<K, V>> pulsarSchema(
- Schema<KeyValue<K, V>> schema, Class<K> keyClass, Class<V> valueClass) {
- PulsarSchema<KeyValue<K, V>> pulsarSchema =
- new PulsarSchema<>(schema, keyClass, valueClass);
- return new PulsarSchemaWrapper<>(pulsarSchema);
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchemaWrapper.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchemaWrapper.java
deleted file mode 100644
index 716d2db8ee1..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/serializer/PulsarSerializationSchemaWrapper.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.serializer;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessage;
-import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
-
-import org.apache.pulsar.client.api.Schema;
-
-/** Wrap the Flink's SerializationSchema into PulsarSerializationSchema. */
-@Internal
-public class PulsarSerializationSchemaWrapper<IN> implements PulsarSerializationSchema<IN> {
- private static final long serialVersionUID = 4948155843623161119L;
-
- private final SerializationSchema<IN> serializationSchema;
-
- public PulsarSerializationSchemaWrapper(SerializationSchema<IN> serializationSchema) {
- this.serializationSchema = serializationSchema;
- }
-
- @Override
- public void open(
- InitializationContext initializationContext,
- PulsarSinkContext sinkContext,
- SinkConfiguration sinkConfiguration)
- throws Exception {
- serializationSchema.open(initializationContext);
- }
-
- @Override
- public PulsarMessage<?> serialize(IN element, PulsarSinkContext sinkContext) {
- PulsarMessageBuilder<byte[]> builder = new PulsarMessageBuilder<>();
- byte[] value = serializationSchema.serialize(element);
- builder.value(Schema.BYTES, value);
-
- return builder.build();
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicMetadataListener.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicMetadataListener.java
deleted file mode 100644
index 7db42e6e4b5..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicMetadataListener.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.topic;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.operators.ProcessingTimeService;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-
-import org.apache.flink.shaded.guava30.com.google.common.base.Objects;
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
-
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.emptyList;
-import static org.apache.flink.connector.pulsar.common.config.PulsarClientFactory.createAdmin;
-import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin;
-import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.isPartition;
-import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicName;
-import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
-import static org.apache.pulsar.common.partition.PartitionedTopicMetadata.NON_PARTITIONED;
-
-/**
- * We need the latest topic metadata for making sure the newly created topic partitions would be
- * used by the Pulsar sink. This routing policy would be different compared with Pulsar Client
- * built-in logic. We use Flink's ProcessingTimer as the executor.
- */
-@Internal
-public class TopicMetadataListener implements Serializable, Closeable {
- private static final long serialVersionUID = 6186948471557507522L;
-
- private static final Logger LOG = LoggerFactory.getLogger(TopicMetadataListener.class);
-
- private final ImmutableList<String> partitionedTopics;
- private final Map<String, Integer> topicMetadata;
- private volatile ImmutableList<String> availableTopics;
-
- // Dynamic fields.
- private transient PulsarAdmin pulsarAdmin;
- private transient Long topicMetadataRefreshInterval;
- private transient ProcessingTimeService timeService;
-
- public TopicMetadataListener() {
- this(emptyList());
- }
-
- public TopicMetadataListener(List<String> topics) {
- List<String> partitions = new ArrayList<>(topics.size());
- Map<String, Integer> metadata = new HashMap<>(topics.size());
- for (String topic : topics) {
- if (isPartition(topic)) {
- partitions.add(topic);
- } else {
- // This would be updated when open writing.
- metadata.put(topic, -1);
- }
- }
-
- this.partitionedTopics = ImmutableList.copyOf(partitions);
- this.topicMetadata = metadata;
- this.availableTopics = ImmutableList.of();
- }
-
- /** Register the topic metadata update in process time service. */
- public void open(SinkConfiguration sinkConfiguration, ProcessingTimeService timeService) {
- if (topicMetadata.isEmpty()) {
- LOG.info("No topics have been provided, skip listener initialize.");
- return;
- }
-
- // Initialize listener properties.
- this.pulsarAdmin = createAdmin(sinkConfiguration);
- this.topicMetadataRefreshInterval = sinkConfiguration.getTopicMetadataRefreshInterval();
- this.timeService = timeService;
-
- // Initialize the topic metadata. Quit if fail to connect to Pulsar.
- sneakyAdmin(this::updateTopicMetadata);
-
- // Register time service.
- triggerNextTopicMetadataUpdate(true);
- }
-
- /**
- * Return all the available topic partitions. We would recalculate the partitions if the topic
- * metadata has been changed. Otherwise, we would return the cached result for better
- * performance.
- */
- public List<String> availableTopics() {
- if (availableTopics.isEmpty()
- && (!partitionedTopics.isEmpty() || !topicMetadata.isEmpty())) {
- List<String> results = new ArrayList<>();
- for (Map.Entry<String, Integer> entry : topicMetadata.entrySet()) {
- int partitionNums = entry.getValue();
- // Get all topics from partitioned and non-partitioned topic names
- if (partitionNums == NON_PARTITIONED) {
- results.add(topicName(entry.getKey()));
- } else {
- for (int i = 0; i < partitionNums; i++) {
- results.add(topicNameWithPartition(entry.getKey(), i));
- }
- }
- }
-
- results.addAll(partitionedTopics);
- this.availableTopics = ImmutableList.copyOf(results);
- }
-
- return availableTopics;
- }
-
- @Override
- public void close() throws IOException {
- if (pulsarAdmin != null) {
- pulsarAdmin.close();
- }
- }
-
- private void triggerNextTopicMetadataUpdate(boolean initial) {
- if (!initial) {
- // We should update the topic metadata, ignore the pulsar admin exception.
- try {
- updateTopicMetadata();
- } catch (PulsarAdminException e) {
- LOG.warn("", e);
- }
- }
-
- // Register next timer.
- long currentProcessingTime = timeService.getCurrentProcessingTime();
- long triggerTime = currentProcessingTime + topicMetadataRefreshInterval;
- timeService.registerTimer(triggerTime, time -> triggerNextTopicMetadataUpdate(false));
- }
-
- private void updateTopicMetadata() throws PulsarAdminException {
- boolean shouldUpdate = false;
-
- for (Map.Entry<String, Integer> entry : topicMetadata.entrySet()) {
- String topic = entry.getKey();
- PartitionedTopicMetadata metadata =
- pulsarAdmin.topics().getPartitionedTopicMetadata(topic);
-
- // Update topic metadata if it has been changed.
- if (!Objects.equal(entry.getValue(), metadata.partitions)) {
- entry.setValue(metadata.partitions);
- shouldUpdate = true;
- }
- }
-
- // Clear available topics if the topic metadata has been changed.
- if (shouldUpdate) {
- this.availableTopics = ImmutableList.of();
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicProducerRegister.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicProducerRegister.java
deleted file mode 100644
index cf8ff00e597..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/topic/TopicProducerRegister.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.sink.writer.topic;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.pulsar.common.metrics.ProducerMetricsInterceptor;
-import org.apache.flink.connector.pulsar.sink.committer.PulsarCommittable;
-import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.flink.shaded.guava30.com.google.common.io.Closer;
-
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerBuilder;
-import org.apache.pulsar.client.api.ProducerStats;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.TypedMessageBuilder;
-import org.apache.pulsar.client.api.transaction.Transaction;
-import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient;
-import org.apache.pulsar.client.api.transaction.TxnID;
-import org.apache.pulsar.client.impl.PulsarClientImpl;
-import org.apache.pulsar.common.schema.SchemaInfo;
-import org.apache.pulsar.shade.com.google.common.base.Strings;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import static org.apache.flink.connector.pulsar.common.config.PulsarClientFactory.createClient;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.NUM_ACKS_RECEIVED;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.NUM_BYTES_SENT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.NUM_MSGS_SENT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.NUM_SEND_FAILED;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.PENDING_QUEUE_SIZE;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.PULSAR_PRODUCER_METRIC_NAME;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_BYTES_RATE;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_50_PCT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_75_PCT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_95_PCT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_999_PCT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_99_PCT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_LATENCY_MILLIS_MAX;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.SEND_MSGS_RATE;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.TOTAL_ACKS_RECEIVED;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.TOTAL_BYTES_SENT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.TOTAL_MSGS_SENT;
-import static org.apache.flink.connector.pulsar.common.metrics.MetricNames.TOTAL_SEND_FAILED;
-import static org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient;
-import static org.apache.flink.connector.pulsar.common.utils.PulsarTransactionUtils.createTransaction;
-import static org.apache.flink.connector.pulsar.sink.config.PulsarSinkConfigUtils.createProducerBuilder;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * All the Pulsar Producers share the same Client, but self hold the queue for a specified topic. So
- * we have to create different instances for different topics.
- */
-@Internal
-public class TopicProducerRegister implements Closeable {
-
- private final PulsarClient pulsarClient;
- private final SinkConfiguration sinkConfiguration;
- private final SinkWriterMetricGroup metricGroup;
- private final Map<String, Map<SchemaInfo, Producer<?>>> producerRegister;
- private final Map<String, Transaction> transactionRegister;
-
- public TopicProducerRegister(
- SinkConfiguration sinkConfiguration, SinkWriterMetricGroup metricGroup) {
- this.pulsarClient = createClient(sinkConfiguration);
- this.sinkConfiguration = sinkConfiguration;
- this.metricGroup = metricGroup;
- this.producerRegister = new HashMap<>();
- this.transactionRegister = new HashMap<>();
-
- if (sinkConfiguration.isEnableMetrics()) {
- metricGroup.setCurrentSendTimeGauge(this::currentSendTimeGauge);
- }
- }
-
- /**
- * Create a TypedMessageBuilder which could be sent to Pulsar directly. First, we would create a
- * topic-related producer or use a cached instead. Then we would try to find a topic-related
- * transaction. We would generate a transaction instance if there is no transaction. Finally, we
- * create the message builder and put the element into it.
- */
- public <T> TypedMessageBuilder<T> createMessageBuilder(String topic, Schema<T> schema) {
- Producer<T> producer = getOrCreateProducer(topic, schema);
- DeliveryGuarantee deliveryGuarantee = sinkConfiguration.getDeliveryGuarantee();
-
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- Transaction transaction = getOrCreateTransaction(topic);
- return producer.newMessage(transaction);
- } else {
- return producer.newMessage();
- }
- }
-
- /**
- * Convert the transactions into a committable list for Pulsar Committer. The transactions would
- * be removed until Flink triggered a checkpoint.
- */
- public List<PulsarCommittable> prepareCommit() {
- List<PulsarCommittable> committables = new ArrayList<>(transactionRegister.size());
- transactionRegister.forEach(
- (topic, transaction) -> {
- TxnID txnID = transaction.getTxnID();
- PulsarCommittable committable = new PulsarCommittable(txnID, topic);
- committables.add(committable);
- });
-
- clearTransactions();
- return committables;
- }
-
- /**
- * Flush all the messages buffered in the client and wait until all messages have been
- * successfully persisted.
- */
- public void flush() throws IOException {
- Collection<Map<SchemaInfo, Producer<?>>> collection = producerRegister.values();
- for (Map<SchemaInfo, Producer<?>> producers : collection) {
- for (Producer<?> producer : producers.values()) {
- producer.flush();
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- try (Closer closer = Closer.create()) {
- // Flush all the pending messages to Pulsar. This wouldn't cause exception.
- closer.register(this::flush);
-
- // Abort all the existed transactions.
- closer.register(this::abortTransactions);
-
- // Remove all the producers.
- closer.register(producerRegister::clear);
-
- // All the producers would be closed by this method.
- // We would block until all the producers have been successfully closed.
- closer.register(pulsarClient);
- }
- }
-
- /** Create or return the cached topic-related producer. */
- @SuppressWarnings("unchecked")
- private <T> Producer<T> getOrCreateProducer(String topic, Schema<T> schema) {
- Map<SchemaInfo, Producer<?>> producers =
- producerRegister.computeIfAbsent(topic, key -> new HashMap<>());
- SchemaInfo schemaInfo = schema.getSchemaInfo();
-
- if (producers.containsKey(schemaInfo)) {
- return (Producer<T>) producers.get(schemaInfo);
- } else {
- ProducerBuilder<T> builder =
- createProducerBuilder(pulsarClient, schema, sinkConfiguration);
- // Set the required topic name.
- builder.topic(topic);
- // Set the sending counter for metrics.
- builder.intercept(new ProducerMetricsInterceptor(metricGroup));
-
- Producer<T> producer = sneakyClient(builder::create);
-
- // Expose the stats for calculating and monitoring.
- exposeProducerMetrics(producer);
- producers.put(schemaInfo, producer);
-
- return producer;
- }
- }
-
- /**
- * Get the cached topic-related transaction. Or create a new transaction after checkpointing.
- */
- private Transaction getOrCreateTransaction(String topic) {
- return transactionRegister.computeIfAbsent(
- topic,
- t -> {
- long timeoutMillis = sinkConfiguration.getTransactionTimeoutMillis();
- return createTransaction(pulsarClient, timeoutMillis);
- });
- }
-
- /** Abort the existed transactions. This method would be used when closing PulsarWriter. */
- private void abortTransactions() {
- if (transactionRegister.isEmpty()) {
- return;
- }
-
- TransactionCoordinatorClient coordinatorClient =
- ((PulsarClientImpl) pulsarClient).getTcClient();
- // This null check is used for making sure transaction is enabled in client.
- checkNotNull(coordinatorClient);
-
- try (Closer closer = Closer.create()) {
- for (Transaction transaction : transactionRegister.values()) {
- TxnID txnID = transaction.getTxnID();
- closer.register(() -> coordinatorClient.abort(txnID));
- }
-
- clearTransactions();
- } catch (IOException e) {
- throw new FlinkRuntimeException(e);
- }
- }
-
- /**
- * Clean these transactions. All transactions should be passed to Pulsar committer, we would
- * create new transaction when new message comes.
- */
- private void clearTransactions() {
- transactionRegister.clear();
- }
-
- private Long currentSendTimeGauge() {
- double sendTime =
- producerRegister.values().stream()
- .flatMap(v -> v.values().stream())
- .map(Producer::getStats)
- .mapToDouble(ProducerStats::getSendLatencyMillis50pct)
- .average()
- .orElse(Long.MAX_VALUE);
-
- return Math.round(sendTime);
- }
-
- private void exposeProducerMetrics(Producer<?> producer) {
- if (sinkConfiguration.isEnableMetrics()) {
- String producerIdentity = producer.getProducerName();
- if (Strings.isNullOrEmpty(producerIdentity)) {
- // Fallback to use the topic name.
- producerIdentity = UUID.randomUUID().toString();
- }
-
- MetricGroup group =
- metricGroup
- .addGroup(PULSAR_PRODUCER_METRIC_NAME)
- .addGroup(producer.getTopic())
- .addGroup(producerIdentity);
- ProducerStats stats = producer.getStats();
-
- group.gauge(NUM_MSGS_SENT, stats::getNumMsgsSent);
- group.gauge(NUM_BYTES_SENT, stats::getNumBytesSent);
- group.gauge(NUM_SEND_FAILED, stats::getNumSendFailed);
- group.gauge(NUM_ACKS_RECEIVED, stats::getNumAcksReceived);
- group.gauge(SEND_MSGS_RATE, stats::getSendMsgsRate);
- group.gauge(SEND_BYTES_RATE, stats::getSendBytesRate);
- group.gauge(SEND_LATENCY_MILLIS_50_PCT, stats::getSendLatencyMillis50pct);
- group.gauge(SEND_LATENCY_MILLIS_75_PCT, stats::getSendLatencyMillis75pct);
- group.gauge(SEND_LATENCY_MILLIS_95_PCT, stats::getSendLatencyMillis95pct);
- group.gauge(SEND_LATENCY_MILLIS_99_PCT, stats::getSendLatencyMillis99pct);
- group.gauge(SEND_LATENCY_MILLIS_999_PCT, stats::getSendLatencyMillis999pct);
- group.gauge(SEND_LATENCY_MILLIS_MAX, stats::getSendLatencyMillisMax);
- group.gauge(TOTAL_MSGS_SENT, stats::getTotalMsgsSent);
- group.gauge(TOTAL_BYTES_SENT, stats::getTotalBytesSent);
- group.gauge(TOTAL_SEND_FAILED, stats::getTotalSendFailed);
- group.gauge(TOTAL_ACKS_RECEIVED, stats::getTotalAcksReceived);
- group.gauge(PENDING_QUEUE_SIZE, stats::getPendingQueueSize);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/PulsarSource.java b/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/PulsarSource.java
deleted file mode 100644
index 0786062a60e..00000000000
--- a/flink-connectors/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/PulsarSource.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.pulsar.source;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.connector.source.Boundedness;
-import org.apache.flink.api.connector.source.Source;
-import org.apache.flink.api.connector.source.SourceReader;
-import org.apache.flink.api.connector.source.SourceReaderContext;
-import org.apache.flink.api.connector.source.SplitEnumerator;
-import org.apache.flink.api.connector.source.SplitEnumeratorContext;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
-import org.apache.flink.connector.pulsar.source.enumerator.PulsarSourceEnumState;
-import org.apache.flink.connector.pulsar.source.enumerator.PulsarSourceEnumStateSerializer;
-import org.apache.flink.connector.pulsar.source.enumerator.PulsarSourceEnumerator;
-import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
-import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
-import org.apache.flink.connector.pulsar.source.enumerator.subscriber.PulsarSubscriber;
-import org.apache.flink.connector.pulsar.source.enumerator.topic.range.RangeGenerator;
-import org.apache.flink.connector.pulsar.source.reader.PulsarSourceReaderFactory;
-import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
-import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchemaInitializationContext;
-import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
-import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplitSerializer;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-/**
- * The Source implementation of Pulsar. Please use a {@link PulsarSourceBuilder} to construct a
- * {@link PulsarSource}. The following example shows how to create a PulsarSource emitting records
- * of <code>String</code> type.
- *
- * <pre>{@code
- * PulsarSource<String> source = PulsarSource
- * .builder()
- * .setTopics(TOPIC1， TOPIC2)
- * .setServiceUrl(getServiceUrl())
- * .setAdminUrl(getAdminUrl())
- * .setSubscriptionName("test")
- * .setDeserializationSchema(PulsarDeserializationSchema.flinkSchema(new SimpleStringSchema()))
- * .setBounded(StopCursor::defaultStopCursor)
- * .build();
- * }</pre>
- *
- * <p>See {@link PulsarSourceBuilder} for more details.
- *
- * @param <OUT> The output type of the source.
- */
-@PublicEvolving
-public final class PulsarSource<OUT>
- implements Source<OUT, PulsarPartitionSplit, PulsarSourceEnumState>,
- ResultTypeQueryable<OUT> {
- private static final long serialVersionUID = 7773108631275567433L;
-
- /**
- * The configuration for pulsar source, we don't support the pulsar's configuration class
- * directly.
- */
- private final SourceConfiguration sourceConfiguration;
-
- private final PulsarSubscriber subscriber;
-
- private final RangeGenerator rangeGenerator;
-
- private final StartCursor startCursor;
-
- private final StopCursor stopCursor;
-
- private final Boundedness boundedness;
-
- /** The pulsar deserialization schema used for deserializing message. */
- private final PulsarDeserializationSchema<OUT> deserializationSchema;
-
- /**
- * The constructor for PulsarSource, it's package protected for forcing using {@link
- * PulsarSourceBuilder}.
- */
- PulsarSource(
- SourceConfiguration sourceConfiguration,
- PulsarSubscriber subscriber,
- RangeGenerator rangeGenerator,
- StartCursor startCursor,
- StopCursor stopCursor,
- Boundedness boundedness,
- PulsarDeserializationSchema<OUT> deserializationSchema) {
- this.sourceConfiguration = sourceConfiguration;
- this.subscriber = subscriber;
- this.rangeGenerator = rangeGenerator;
- this.startCursor = startCursor;
- this.stopCursor = stopCursor;
- this.boundedness = boundedness;
- this.deserializationSchema = deserializationSchema;
- }
-
- /**
- * Get a PulsarSourceBuilder to builder a {@link PulsarSource}.
- *
- * @return a Pulsar source builder.
- */
- public static <OUT> PulsarSourceBuilder<OUT> builder() {
- return new PulsarSourceBuilder<>();
- }
-
- @Override
- public Boundedness getBoundedness() {
- return boundedness;
- }
-
- @Internal
- @Override
- public SourceReader<OUT, PulsarPartitionSplit> createReader(SourceReaderContext readerContext)
- throws Exception {
- // Initialize the deserialization schema before creating the pulsar reader.
- PulsarDeserializationSchemaInitializationContext initializationContext =
- new PulsarDeserializationSchemaInitializationContext(readerContext);
- deserializationSchema.open(initializationContext, sourceConfiguration);
-
- return PulsarSourceReaderFactory.create(
- readerContext, deserializationSchema, sourceConfiguration);
- }
-
- @Internal
- @Override
- public SplitEnumerator<PulsarPartitionSplit, PulsarSourceEnumState> createEnumerator(
- SplitEnumeratorContext<PulsarPartitionSplit> enumContext) {
- return new PulsarSourceEnumerator(
- subscriber,
- startCursor,
- stopCursor,
- rangeGenerator,
- sourceConfiguration,
- enumContext);
- }
-
- @Internal
- @Override
- public SplitEnumerator<PulsarPartitionSplit, PulsarSourceEnumState> restoreEnumerator(
- SplitEnumeratorContext<PulsarPartitionSplit> enumContext,
- PulsarSourceEnumState checkpoint) {
- return new PulsarSourceEnumerator(
- subscriber,
- startCursor,
- stopCursor,
- rangeGenerator,
- sourceConfiguration,
- enumContext,
- checkpoint);
- }
-
- @Internal
... 17408 lines suppressed ...