You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2023/06/01 22:22:27 UTC
[flink] branch remove-kafka-connector created (now 342a89176bd)
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a change to branch remove-kafka-connector
in repository https://gitbox.apache.org/repos/asf/flink.git
at 342a89176bd [WIP] Remove all Kafka connector code from main repo
This branch includes the following new commits:
new 342a89176bd [WIP] Remove all Kafka connector code from main repo
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[flink] 01/01: [WIP] Remove all Kafka connector code from main repo
Posted by tz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch remove-kafka-connector
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 342a89176bdf5d3650629bfc4e401641c9ba65ef
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Thu Jun 1 15:21:41 2023 -0700
[WIP] Remove all Kafka connector code from main repo
---
flink-architecture-tests/pom.xml | 2 +-
.../8248e31e-b706-4471-bb65-4c927c65fccf | 0
.../97dda445-f6bc-43e2-8106-5876ca0cd052 | 78 -
.../archunit-violations/stored.rules | 4 -
flink-connectors/flink-connector-kafka/pom.xml | 332 ---
.../apache/flink/connector/kafka/MetricUtil.java | 79 -
.../kafka/sink/DefaultKafkaSinkContext.java | 79 -
.../kafka/sink/FlinkKafkaInternalProducer.java | 375 ---
.../connector/kafka/sink/KafkaCommittable.java | 104 -
.../kafka/sink/KafkaCommittableSerializer.java | 57 -
.../flink/connector/kafka/sink/KafkaCommitter.java | 146 --
.../kafka/sink/KafkaRecordSerializationSchema.java | 102 -
.../KafkaRecordSerializationSchemaBuilder.java | 336 ---
.../kafka/sink/KafkaSerializerWrapper.java | 91 -
.../flink/connector/kafka/sink/KafkaSink.java | 138 -
.../connector/kafka/sink/KafkaSinkBuilder.java | 215 --
.../flink/connector/kafka/sink/KafkaWriter.java | 473 ----
.../connector/kafka/sink/KafkaWriterState.java | 60 -
.../kafka/sink/KafkaWriterStateSerializer.java | 54 -
.../flink/connector/kafka/sink/Recyclable.java | 49 -
.../flink/connector/kafka/sink/TopicSelector.java | 31 -
.../connector/kafka/sink/TransactionAborter.java | 131 -
.../kafka/sink/TransactionalIdFactory.java | 41 -
.../kafka/sink/TransactionsToAbortChecker.java | 95 -
.../flink/connector/kafka/source/KafkaSource.java | 241 --
.../connector/kafka/source/KafkaSourceBuilder.java | 534 ----
.../connector/kafka/source/KafkaSourceOptions.java | 65 -
.../source/enumerator/KafkaSourceEnumState.java | 39 -
.../enumerator/KafkaSourceEnumStateSerializer.java | 124 -
.../source/enumerator/KafkaSourceEnumerator.java | 634 -----
.../initializer/NoStoppingOffsetsInitializer.java | 52 -
.../enumerator/initializer/OffsetsInitializer.java | 186 --
.../initializer/OffsetsInitializerValidator.java | 39 -
.../ReaderHandledOffsetsInitializer.java | 83 -
.../initializer/SpecifiedOffsetsInitializer.java | 108 -
.../initializer/TimestampOffsetsInitializer.java | 76 -
.../enumerator/subscriber/KafkaSubscriber.java | 67 -
.../subscriber/KafkaSubscriberUtils.java | 50 -
.../subscriber/PartitionSetSubscriber.java | 76 -
.../enumerator/subscriber/TopicListSubscriber.java | 63 -
.../subscriber/TopicPatternSubscriber.java | 65 -
.../source/metrics/KafkaSourceReaderMetrics.java | 330 ---
.../source/reader/KafkaPartitionSplitReader.java | 532 ----
.../kafka/source/reader/KafkaRecordEmitter.java | 81 -
.../kafka/source/reader/KafkaSourceReader.java | 199 --
.../KafkaDeserializationSchemaWrapper.java | 65 -
.../KafkaRecordDeserializationSchema.java | 129 -
...KafkaValueOnlyDeserializationSchemaWrapper.java | 58 -
.../KafkaValueOnlyDeserializerWrapper.java | 106 -
.../reader/fetcher/KafkaSourceFetcherManager.java | 112 -
.../kafka/source/split/KafkaPartitionSplit.java | 152 --
.../split/KafkaPartitionSplitSerializer.java | 72 -
.../source/split/KafkaPartitionSplitState.java | 56 -
.../connectors/kafka/FlinkKafkaConsumer.java | 342 ---
.../connectors/kafka/FlinkKafkaConsumerBase.java | 1227 ---------
.../connectors/kafka/FlinkKafkaErrorCode.java | 27 -
.../connectors/kafka/FlinkKafkaException.java | 44 -
.../connectors/kafka/FlinkKafkaProducer.java | 1950 --------------
.../connectors/kafka/FlinkKafkaProducer011.java | 72 -
.../connectors/kafka/FlinkKafkaProducerBase.java | 442 ----
.../connectors/kafka/KafkaContextAware.java | 55 -
.../kafka/KafkaDeserializationSchema.java | 84 -
.../connectors/kafka/KafkaSerializationSchema.java | 61 -
.../connectors/kafka/config/BoundedMode.java | 49 -
.../connectors/kafka/config/OffsetCommitMode.java | 42 -
.../connectors/kafka/config/OffsetCommitModes.java | 52 -
.../connectors/kafka/config/StartupMode.java | 62 -
.../kafka/internals/AbstractFetcher.java | 611 -----
.../internals/AbstractPartitionDiscoverer.java | 256 --
.../kafka/internals/ClosableBlockingQueue.java | 500 ----
.../connectors/kafka/internals/ExceptionProxy.java | 123 -
.../internals/FlinkKafkaInternalProducer.java | 421 ---
.../connectors/kafka/internals/Handover.java | 212 --
.../kafka/internals/KafkaCommitCallback.java | 44 -
.../kafka/internals/KafkaConsumerThread.java | 564 ----
.../KafkaDeserializationSchemaWrapper.java | 70 -
.../connectors/kafka/internals/KafkaFetcher.java | 260 --
.../kafka/internals/KafkaPartitionDiscoverer.java | 114 -
.../internals/KafkaSerializationSchemaWrapper.java | 112 -
.../kafka/internals/KafkaShuffleFetcher.java | 305 ---
.../kafka/internals/KafkaTopicPartition.java | 137 -
.../internals/KafkaTopicPartitionAssigner.java | 61 -
.../kafka/internals/KafkaTopicPartitionLeader.java | 107 -
.../kafka/internals/KafkaTopicPartitionState.java | 132 -
.../KafkaTopicPartitionStateSentinel.java | 62 -
...aTopicPartitionStateWithWatermarkGenerator.java | 99 -
.../kafka/internals/KafkaTopicsDescriptor.java | 91 -
.../internals/KeyedSerializationSchemaWrapper.java | 58 -
.../SourceContextWatermarkOutputAdapter.java | 50 -
.../kafka/internals/TransactionalIdsGenerator.java | 100 -
.../metrics/KafkaConsumerMetricConstants.java | 56 -
.../metrics/KafkaMetricMutableWrapper.java | 48 -
.../internals/metrics/KafkaMetricWrapper.java | 42 -
.../kafka/partitioner/FlinkFixedPartitioner.java | 93 -
.../kafka/partitioner/FlinkKafkaPartitioner.java | 56 -
.../kafka/shuffle/FlinkKafkaShuffle.java | 402 ---
.../kafka/shuffle/FlinkKafkaShuffleConsumer.java | 93 -
.../kafka/shuffle/FlinkKafkaShuffleProducer.java | 229 --
.../kafka/shuffle/StreamKafkaShuffleSink.java | 43 -
.../table/DynamicKafkaDeserializationSchema.java | 273 --
.../DynamicKafkaRecordSerializationSchema.java | 177 --
.../kafka/table/KafkaConnectorOptions.java | 355 ---
.../kafka/table/KafkaConnectorOptionsUtil.java | 697 -----
.../connectors/kafka/table/KafkaDynamicSink.java | 479 ----
.../connectors/kafka/table/KafkaDynamicSource.java | 678 -----
.../kafka/table/KafkaDynamicTableFactory.java | 449 ----
.../connectors/kafka/table/ReducingUpsertSink.java | 91 -
.../kafka/table/ReducingUpsertWriter.java | 183 --
.../kafka/table/SinkBufferFlushMode.java | 80 -
.../table/UpsertKafkaDynamicTableFactory.java | 406 ---
.../JSONKeyValueDeserializationSchema.java | 89 -
.../serialization/KeyedDeserializationSchema.java | 56 -
.../serialization/KeyedSerializationSchema.java | 61 -
...TypeInformationKeyValueSerializationSchema.java | 205 --
.../org.apache.flink.table.factories.Factory | 17 -
.../architecture/TestCodeArchitectureTest.java | 43 -
.../sink/FlinkKafkaInternalProducerITCase.java | 156 --
.../kafka/sink/KafkaCommittableSerializerTest.java | 44 -
.../connector/kafka/sink/KafkaCommitterTest.java | 124 -
.../KafkaRecordSerializationSchemaBuilderTest.java | 355 ---
.../connector/kafka/sink/KafkaSinkBuilderTest.java | 113 -
.../connector/kafka/sink/KafkaSinkITCase.java | 691 -----
.../connector/kafka/sink/KafkaTransactionLog.java | 199 --
.../kafka/sink/KafkaTransactionLogITCase.java | 162 --
.../connector/kafka/sink/KafkaWriterITCase.java | 673 -----
.../kafka/sink/KafkaWriterStateSerializerTest.java | 42 -
.../kafka/sink/TransactionIdFactoryTest.java | 35 -
.../kafka/sink/TransactionToAbortCheckerTest.java | 75 -
.../kafka/sink/testutils/KafkaDataReader.java | 69 -
.../sink/testutils/KafkaSinkExternalContext.java | 264 --
.../testutils/KafkaSinkExternalContextFactory.java | 53 -
.../kafka/source/KafkaSourceBuilderTest.java | 209 --
.../connector/kafka/source/KafkaSourceITCase.java | 532 ----
.../kafka/source/KafkaSourceLegacyITCase.java | 164 --
.../kafka/source/KafkaSourceTestUtils.java | 59 -
.../source/enumerator/KafkaEnumeratorTest.java | 560 ----
.../KafkaSourceEnumStateSerializerTest.java | 108 -
.../initializer/OffsetsInitializerTest.java | 170 --
.../enumerator/subscriber/KafkaSubscriberTest.java | 130 -
.../metrics/KafkaSourceReaderMetricsTest.java | 159 --
.../reader/KafkaPartitionSplitReaderTest.java | 431 ----
.../kafka/source/reader/KafkaSourceReaderTest.java | 635 -----
.../KafkaRecordDeserializationSchemaTest.java | 193 --
.../split/KafkaPartitionSplitSerializerTest.java | 54 -
.../kafka/testutils/KafkaPartitionDataWriter.java | 64 -
.../testutils/KafkaSourceExternalContext.java | 245 --
.../KafkaSourceExternalContextFactory.java | 59 -
.../kafka/testutils/KafkaSourceTestEnv.java | 270 --
.../flink/connector/kafka/testutils/KafkaUtil.java | 189 --
.../kafka/FlinkFixedPartitionerTest.java | 109 -
.../kafka/FlinkKafkaConsumerBaseMigrationTest.java | 425 ---
.../kafka/FlinkKafkaConsumerBaseTest.java | 1523 -----------
.../kafka/FlinkKafkaInternalProducerITCase.java | 261 --
.../kafka/FlinkKafkaProducerBaseTest.java | 482 ----
.../connectors/kafka/FlinkKafkaProducerITCase.java | 833 ------
.../FlinkKafkaProducerMigrationOperatorTest.java | 58 -
.../kafka/FlinkKafkaProducerMigrationTest.java | 81 -
.../FlinkKafkaProducerStateSerializerTest.java | 148 --
.../connectors/kafka/FlinkKafkaProducerTest.java | 164 --
.../JSONKeyValueDeserializationSchemaTest.java | 130 -
.../connectors/kafka/KafkaConsumerTestBase.java | 2728 --------------------
.../streaming/connectors/kafka/KafkaITCase.java | 380 ---
.../connectors/kafka/KafkaMigrationTestBase.java | 158 --
.../kafka/KafkaProducerAtLeastOnceITCase.java | 43 -
.../kafka/KafkaProducerExactlyOnceITCase.java | 38 -
.../connectors/kafka/KafkaProducerTestBase.java | 401 ---
.../kafka/KafkaSerializerUpgradeTest.java | 172 --
.../kafka/KafkaShortRetentionTestBase.java | 229 --
.../streaming/connectors/kafka/KafkaTestBase.java | 341 ---
.../connectors/kafka/KafkaTestBaseWithFlink.java | 41 -
.../connectors/kafka/KafkaTestEnvironment.java | 222 --
.../connectors/kafka/KafkaTestEnvironmentImpl.java | 497 ----
.../NextTransactionalIdHintSerializerTest.java | 56 -
.../kafka/internals/AbstractFetcherTest.java | 279 --
.../internals/AbstractFetcherWatermarksTest.java | 499 ----
.../internals/AbstractPartitionDiscovererTest.java | 561 ----
.../kafka/internals/ClosableBlockingQueueTest.java | 616 -----
.../kafka/internals/KafkaTopicPartitionTest.java | 57 -
.../kafka/internals/KafkaTopicsDescriptorTest.java | 66 -
.../metrics/KafkaMetricMutableWrapperTest.java | 99 -
.../shuffle/KafkaShuffleExactlyOnceITCase.java | 218 --
.../kafka/shuffle/KafkaShuffleITCase.java | 547 ----
.../kafka/shuffle/KafkaShuffleTestBase.java | 313 ---
.../kafka/table/KafkaChangelogTableITCase.java | 484 ----
.../kafka/table/KafkaConnectorOptionsUtilTest.java | 138 -
.../kafka/table/KafkaDynamicTableFactoryTest.java | 1340 ----------
.../connectors/kafka/table/KafkaTableITCase.java | 1169 ---------
.../connectors/kafka/table/KafkaTableTestBase.java | 238 --
.../kafka/table/KafkaTableTestUtils.java | 110 -
.../kafka/table/ReducingUpsertWriterTest.java | 385 ---
.../table/UpsertKafkaDynamicTableFactoryTest.java | 666 -----
.../kafka/table/UpsertKafkaTableITCase.java | 882 -------
.../kafka/testutils/ClusterCommunicationUtils.java | 53 -
.../connectors/kafka/testutils/DataGenerators.java | 244 --
.../kafka/testutils/FailingIdentityMapper.java | 129 -
.../testutils/FakeStandardProducerConfig.java | 35 -
.../connectors/kafka/testutils/IntegerSource.java | 133 -
.../kafka/testutils/PartitionValidatingMapper.java | 57 -
.../kafka/testutils/TestPartitionDiscoverer.java | 138 -
.../kafka/testutils/TestSourceContext.java | 85 -
.../kafka/testutils/ThrottledMapper.java | 43 -
.../kafka/testutils/Tuple2FlinkPartitioner.java | 50 -
.../kafka/testutils/ValidatingExactlyOnceSink.java | 146 --
.../src/test/resources/archunit.properties | 31 -
.../src/test/resources/canal-data.txt | 11 -
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.11/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.12/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.13/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.14/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.15/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.16/test-data | Bin 19 -> 0 bytes
.../serializer-snapshot | Bin 126 -> 0 bytes
.../context-state-serializer-1.17/test-data | Bin 19 -> 0 bytes
.../resources/debezium-data-schema-exclude.txt | 16 -
...11-migration-kafka-producer-flink-1.10-snapshot | Bin 2032 -> 0 bytes
...11-migration-kafka-producer-flink-1.11-snapshot | Bin 2040 -> 0 bytes
....11-migration-kafka-producer-flink-1.8-snapshot | Bin 2032 -> 0 bytes
....11-migration-kafka-producer-flink-1.9-snapshot | Bin 2032 -> 0 bytes
...r-migration-test-flink1.10-empty-state-snapshot | Bin 1330 -> 0 bytes
...afka-consumer-migration-test-flink1.10-snapshot | Bin 1384 -> 0 bytes
...r-migration-test-flink1.11-empty-state-snapshot | Bin 1151 -> 0 bytes
...afka-consumer-migration-test-flink1.11-snapshot | Bin 1205 -> 0 bytes
...r-migration-test-flink1.12-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.12-snapshot | Bin 1519 -> 0 bytes
...r-migration-test-flink1.13-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.13-snapshot | Bin 1519 -> 0 bytes
...r-migration-test-flink1.14-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.14-snapshot | Bin 1519 -> 0 bytes
...r-migration-test-flink1.15-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.15-snapshot | Bin 1519 -> 0 bytes
...r-migration-test-flink1.16-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.16-snapshot | Bin 1519 -> 0 bytes
...r-migration-test-flink1.17-empty-state-snapshot | Bin 1465 -> 0 bytes
...afka-consumer-migration-test-flink1.17-snapshot | Bin 1519 -> 0 bytes
...er-migration-test-flink1.8-empty-state-snapshot | Bin 1330 -> 0 bytes
...kafka-consumer-migration-test-flink1.8-snapshot | Bin 1384 -> 0 bytes
...er-migration-test-flink1.9-empty-state-snapshot | Bin 1330 -> 0 bytes
...kafka-consumer-migration-test-flink1.9-snapshot | Bin 1384 -> 0 bytes
...ka-migration-kafka-producer-flink-1.10-snapshot | Bin 1232 -> 0 bytes
...ka-migration-kafka-producer-flink-1.11-snapshot | Bin 1240 -> 0 bytes
...ka-migration-kafka-producer-flink-1.12-snapshot | Bin 1240 -> 0 bytes
...ka-migration-kafka-producer-flink-1.13-snapshot | Bin 1240 -> 0 bytes
...ka-migration-kafka-producer-flink-1.14-snapshot | Bin 1241 -> 0 bytes
...ka-migration-kafka-producer-flink-1.15-snapshot | Bin 1242 -> 0 bytes
...ka-migration-kafka-producer-flink-1.16-snapshot | Bin 1242 -> 0 bytes
...ka-migration-kafka-producer-flink-1.17-snapshot | Bin 1242 -> 0 bytes
...fka-migration-kafka-producer-flink-1.8-snapshot | Bin 1232 -> 0 bytes
...fka-migration-kafka-producer-flink-1.9-snapshot | Bin 1232 -> 0 bytes
.../src/test/resources/log4j2-test.properties | 41 -
.../src/test/resources/maxwell-data.txt | 20 -
.../src/test/resources/product_changelog.txt | 7 -
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.11/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.12/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.13/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.14/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.15/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.16/test-data | Bin 17 -> 0 bytes
.../serializer-snapshot | Bin 134 -> 0 bytes
.../transaction-state-serializer-1.17/test-data | Bin 17 -> 0 bytes
flink-connectors/flink-sql-connector-kafka/pom.xml | 102 -
.../src/main/resources/META-INF/NOTICE | 9 -
.../flink/connectors/kafka/PackagingITCase.java | 40 -
flink-connectors/pom.xml | 2 -
.../flink-confluent-schema-registry/pom.xml | 2 +-
.../flink-end-to-end-tests-common-kafka/pom.xml | 247 --
.../tests/util/kafka/KafkaContainerClient.java | 147 --
.../flink/tests/util/kafka/KafkaResource.java | 131 -
.../tests/util/kafka/KafkaResourceFactory.java | 36 -
.../util/kafka/LocalStandaloneKafkaResource.java | 452 ----
.../kafka/LocalStandaloneKafkaResourceFactory.java | 48 -
.../flink/tests/util/kafka/KafkaSinkE2ECase.java | 83 -
.../flink/tests/util/kafka/KafkaSourceE2ECase.java | 84 -
.../tests/util/kafka/SQLClientKafkaITCase.java | 235 --
.../util/kafka/SQLClientSchemaRegistryITCase.java | 260 --
.../flink/tests/util/kafka/SmokeKafkaITCase.java | 189 --
.../kafka/containers/SchemaRegistryContainer.java | 51 -
.../src/test/resources/kafka_e2e.sql | 75 -
.../src/test/resources/log4j2-test.properties | 34 -
.../flink-sql-client-test/pom.xml | 4 +-
.../flink-streaming-kafka-test-base/pom.xml | 43 -
.../kafka/test/base/CustomWatermarkExtractor.java | 53 -
.../streaming/kafka/test/base/KafkaEvent.java | 74 -
.../kafka/test/base/KafkaEventSchema.java | 54 -
.../kafka/test/base/KafkaExampleUtil.java | 52 -
.../kafka/test/base/RollingAdditionMapper.java | 55 -
.../flink-streaming-kafka-test/pom.xml | 87 -
.../flink/streaming/kafka/test/KafkaExample.java | 80 -
flink-end-to-end-tests/pom.xml | 3 -
flink-examples/flink-examples-streaming/pom.xml | 2 +-
flink-python/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
304 files changed, 8 insertions(+), 51309 deletions(-)
diff --git a/flink-architecture-tests/pom.xml b/flink-architecture-tests/pom.xml
index 0ce107c6a18..aeed3bcd470 100644
--- a/flink-architecture-tests/pom.xml
+++ b/flink-architecture-tests/pom.xml
@@ -177,7 +177,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka</artifactId>
- <version>${project.version}</version>
+ <version>3.0.0-1.17</version>
<scope>test</scope>
</dependency>
diff --git a/flink-connectors/flink-connector-kafka/archunit-violations/8248e31e-b706-4471-bb65-4c927c65fccf b/flink-connectors/flink-connector-kafka/archunit-violations/8248e31e-b706-4471-bb65-4c927c65fccf
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/flink-connectors/flink-connector-kafka/archunit-violations/97dda445-f6bc-43e2-8106-5876ca0cd052 b/flink-connectors/flink-connector-kafka/archunit-violations/97dda445-f6bc-43e2-8106-5876ca0cd052
deleted file mode 100644
index 50c10b7c115..00000000000
--- a/flink-connectors/flink-connector-kafka/archunit-violations/97dda445-f6bc-43e2-8106-5876ca0cd052
+++ /dev/null
@@ -1,78 +0,0 @@
-org.apache.flink.connector.kafka.sink.FlinkKafkaInternalProducerITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.kafka.sink.KafkaSinkITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.kafka.sink.KafkaTransactionLogITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.kafka.sink.KafkaWriterITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.kafka.source.KafkaSourceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.connector.kafka.source.KafkaSourceLegacyITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.FlinkKafkaInternalProducerITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducerITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.KafkaITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.KafkaProducerAtLeastOnceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.KafkaProducerExactlyOnceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.shuffle.KafkaShuffleExactlyOnceITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
-org.apache.flink.streaming.connectors.kafka.shuffle.KafkaShuffleITCase does not satisfy: only one of the following predicates match:\
-* reside in a package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type InternalMiniClusterExtension and annotated with @RegisterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and contain any fields that are static, final, and of type MiniClusterExtension and annotated with @RegisterExtension or are , and of type MiniClusterTestEnvironment and annotated with @TestEnv\
-* reside in a package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class InternalMiniClusterExtension\
-* reside outside of package 'org.apache.flink.runtime.*' and is annotated with @ExtendWith with class MiniClusterExtension\
- or contain any fields that are public, static, and of type MiniClusterWithClientResource and final and annotated with @ClassRule or contain any fields that is of type MiniClusterWithClientResource and public and final and not static and annotated with @Rule
\ No newline at end of file
diff --git a/flink-connectors/flink-connector-kafka/archunit-violations/stored.rules b/flink-connectors/flink-connector-kafka/archunit-violations/stored.rules
deleted file mode 100644
index a9c54e22cc5..00000000000
--- a/flink-connectors/flink-connector-kafka/archunit-violations/stored.rules
+++ /dev/null
@@ -1,4 +0,0 @@
-#
-#Tue Feb 22 12:18:13 CET 2022
-Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=8248e31e-b706-4471-bb65-4c927c65fccf
-ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=97dda445-f6bc-43e2-8106-5876ca0cd052
diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml
deleted file mode 100644
index 607af881a51..00000000000
--- a/flink-connectors/flink-connector-kafka/pom.xml
+++ /dev/null
@@ -1,332 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>flink-connectors</artifactId>
- <groupId>org.apache.flink</groupId>
- <version>1.18-SNAPSHOT</version>
- </parent>
-
- <artifactId>flink-connector-kafka</artifactId>
- <name>Flink : Connectors : Kafka</name>
-
- <packaging>jar</packaging>
-
- <properties>
- <kafka.version>3.2.3</kafka.version>
- </properties>
-
- <dependencies>
-
- <!-- Core -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <!-- Connectors -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!-- Table ecosystem -->
-
- <!-- Projects depending on this project won't depend on flink-table-*. -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-java-bridge</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
-
- <!-- Kafka -->
-
- <!-- Add Kafka 2.x as a dependency -->
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- <version>${kafka.version}</version>
- </dependency>
-
- <!-- Tests -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
-
- <dependency>
- <!-- include 2.0 server for tests -->
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_${scala.binary.version}</artifactId>
- <version>${kafka.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>kafka</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>junit-jupiter</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics-jmx</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <!-- Kafka table factory testing -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-common</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-scala-bridge_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <!-- Kafka SQL IT test with formats -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro-confluent-registry</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-csv</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>fink-migration-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <!-- ArchUit test dependencies -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-architecture-tests-test</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <configuration>
- <includes>
- <include>**/KafkaTestEnvironment*</include>
- <include>**/testutils/*</include>
- <include>META-INF/LICENSE</include>
- <include>META-INF/NOTICE</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-test-sources</id>
- <goals>
- <goal>test-jar-no-fork</goal>
- </goals>
- <configuration>
- <archive>
- <!-- Globally exclude maven metadata, because it may accidentally bundle files we don't intend to -->
- <addMavenDescriptor>false</addMavenDescriptor>
- </archive>
- <includes>
- <include>**/KafkaTestEnvironment*</include>
- <include>**/testutils/*</include>
- <include>META-INF/LICENSE</include>
- <include>META-INF/NOTICE</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>generate-migration-test-data</id>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-migration-test-data</id>
- <phase>package</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <condition property="optional.classes" value="--classes '${generate.classes}'"
- else="">
- <isset property="generate.classes"/>
- </condition>
- <condition property="optional.prefixes"
- value="--prefixes '${generate.prefixes}'" else="">
- <isset property="generate.prefixes"/>
- </condition>
- <java classname="org.apache.flink.test.migration.MigrationTestsSnapshotGenerator"
- fork="true" failonerror="true" dir="${project.basedir}">
- <classpath refid="maven.test.classpath"/>
- <arg value="--dir"/>
- <arg line="${project.basedir}"/>
- <arg value="--version"/>
- <arg value="${generate.version}"/>
- <arg line="${optional.classes}"/>
- <arg line="${optional.prefixes}"/>
- </java>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/MetricUtil.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/MetricUtil.java
deleted file mode 100644
index 330315f2ed7..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/MetricUtil.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.metrics.Counter;
-
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-
-import java.util.Map;
-import java.util.function.Predicate;
-
-/** Collection of methods to interact with Kafka's client metric system. */
-@Internal
-public class MetricUtil {
-
- /**
- * Tries to find the Kafka {@link Metric} in the provided metrics.
- *
- * @return {@link Metric} which exposes continuous updates
- * @throws IllegalStateException if the metric is not part of the provided metrics
- */
- public static Metric getKafkaMetric(
- Map<MetricName, ? extends Metric> metrics, String metricGroup, String metricName) {
- return getKafkaMetric(
- metrics,
- e ->
- e.getKey().group().equals(metricGroup)
- && e.getKey().name().equals(metricName));
- }
-
- /**
- * Tries to find the Kafka {@link Metric} in the provided metrics matching a given filter.
- *
- * @return {@link Metric} which exposes continuous updates
- * @throws IllegalStateException if no metric matches the given filter
- */
- public static Metric getKafkaMetric(
- Map<MetricName, ? extends Metric> metrics,
- Predicate<Map.Entry<MetricName, ? extends Metric>> filter) {
- return metrics.entrySet().stream()
- .filter(filter)
- .map(Map.Entry::getValue)
- .findFirst()
- .orElseThrow(
- () ->
- new IllegalStateException(
- "Cannot find Kafka metric matching current filter."));
- }
-
- /**
- * Ensures that the counter has the same value as the given Kafka metric.
- *
- * <p>Do not use this method for every record because {@link Metric#metricValue()} is an
- * expensive operation.
- *
- * @param from Kafka's {@link Metric} to query
- * @param to {@link Counter} to write the value to
- */
- public static void sync(Metric from, Counter to) {
- to.inc(((Number) from.metricValue()).longValue() - to.getCount());
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/DefaultKafkaSinkContext.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/DefaultKafkaSinkContext.java
deleted file mode 100644
index bec5ab67e81..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/DefaultKafkaSinkContext.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.common.PartitionInfo;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * Context providing information to assist constructing a {@link
- * org.apache.kafka.clients.producer.ProducerRecord}.
- */
-public class DefaultKafkaSinkContext implements KafkaRecordSerializationSchema.KafkaSinkContext {
-
- private final int subtaskId;
- private final int numberOfParallelInstances;
- private final Properties kafkaProducerConfig;
-
- private final Map<String, int[]> cachedPartitions = new HashMap<>();
-
- public DefaultKafkaSinkContext(
- int subtaskId, int numberOfParallelInstances, Properties kafkaProducerConfig) {
- this.subtaskId = subtaskId;
- this.numberOfParallelInstances = numberOfParallelInstances;
- this.kafkaProducerConfig = kafkaProducerConfig;
- }
-
- @Override
- public int getParallelInstanceId() {
- return subtaskId;
- }
-
- @Override
- public int getNumberOfParallelInstances() {
- return numberOfParallelInstances;
- }
-
- @Override
- public int[] getPartitionsForTopic(String topic) {
- return cachedPartitions.computeIfAbsent(topic, this::fetchPartitionsForTopic);
- }
-
- private int[] fetchPartitionsForTopic(String topic) {
- try (final Producer<?, ?> producer = new KafkaProducer<>(kafkaProducerConfig)) {
- // the fetched list is immutable, so we're creating a mutable copy in order to sort
- // it
- final List<PartitionInfo> partitionsList =
- new ArrayList<>(producer.partitionsFor(topic));
-
- return partitionsList.stream()
- .sorted(Comparator.comparing(PartitionInfo::partition))
- .map(PartitionInfo::partition)
- .mapToInt(Integer::intValue)
- .toArray();
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/FlinkKafkaInternalProducer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/FlinkKafkaInternalProducer.java
deleted file mode 100644
index a023cdd144d..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/FlinkKafkaInternalProducer.java
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.internals.TransactionManager;
-import org.apache.kafka.clients.producer.internals.TransactionalRequestResult;
-import org.apache.kafka.common.errors.ProducerFencedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.time.Duration;
-import java.util.Properties;
-
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * A {@link KafkaProducer} that exposes private fields to allow resume producing from a given state.
- */
-class FlinkKafkaInternalProducer<K, V> extends KafkaProducer<K, V> {
-
- private static final Logger LOG = LoggerFactory.getLogger(FlinkKafkaInternalProducer.class);
- private static final String TRANSACTION_MANAGER_FIELD_NAME = "transactionManager";
- private static final String TRANSACTION_MANAGER_STATE_ENUM =
- "org.apache.kafka.clients.producer.internals.TransactionManager$State";
- private static final String PRODUCER_ID_AND_EPOCH_FIELD_NAME = "producerIdAndEpoch";
-
- @Nullable private String transactionalId;
- private volatile boolean inTransaction;
- private volatile boolean closed;
-
- public FlinkKafkaInternalProducer(Properties properties, @Nullable String transactionalId) {
- super(withTransactionalId(properties, transactionalId));
- this.transactionalId = transactionalId;
- }
-
- private static Properties withTransactionalId(
- Properties properties, @Nullable String transactionalId) {
- if (transactionalId == null) {
- return properties;
- }
- Properties props = new Properties();
- props.putAll(properties);
- props.setProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId);
- return props;
- }
-
- @Override
- public void flush() {
- super.flush();
- if (inTransaction) {
- flushNewPartitions();
- }
- }
-
- @Override
- public void beginTransaction() throws ProducerFencedException {
- super.beginTransaction();
- inTransaction = true;
- }
-
- @Override
- public void abortTransaction() throws ProducerFencedException {
- LOG.debug("abortTransaction {}", transactionalId);
- checkState(inTransaction, "Transaction was not started");
- inTransaction = false;
- super.abortTransaction();
- }
-
- @Override
- public void commitTransaction() throws ProducerFencedException {
- LOG.debug("commitTransaction {}", transactionalId);
- checkState(inTransaction, "Transaction was not started");
- inTransaction = false;
- super.commitTransaction();
- }
-
- public boolean isInTransaction() {
- return inTransaction;
- }
-
- @Override
- public void close() {
- closed = true;
- if (inTransaction) {
- // This is state is most likely reached in case of a failure.
- // If this producer is still in transaction, it should be committing.
- // However, at this point, we cannot decide that and we shouldn't prolong cancellation.
- // So hard kill this producer with all resources.
- super.close(Duration.ZERO);
- } else {
- // If this is outside of a transaction, we should be able to cleanly shutdown.
- super.close(Duration.ofHours(1));
- }
- }
-
- @Override
- public void close(Duration timeout) {
- closed = true;
- super.close(timeout);
- }
-
- public boolean isClosed() {
- return closed;
- }
-
- @Nullable
- public String getTransactionalId() {
- return transactionalId;
- }
-
- public short getEpoch() {
- Object transactionManager = getTransactionManager();
- Object producerIdAndEpoch = getField(transactionManager, PRODUCER_ID_AND_EPOCH_FIELD_NAME);
- return (short) getField(producerIdAndEpoch, "epoch");
- }
-
- public long getProducerId() {
- Object transactionManager = getTransactionManager();
- Object producerIdAndEpoch = getField(transactionManager, PRODUCER_ID_AND_EPOCH_FIELD_NAME);
- return (long) getField(producerIdAndEpoch, "producerId");
- }
-
- public void initTransactionId(String transactionalId) {
- if (!transactionalId.equals(this.transactionalId)) {
- setTransactionId(transactionalId);
- initTransactions();
- }
- }
-
- public void setTransactionId(String transactionalId) {
- if (!transactionalId.equals(this.transactionalId)) {
- checkState(
- !inTransaction,
- String.format("Another transaction %s is still open.", transactionalId));
- LOG.debug("Change transaction id from {} to {}", this.transactionalId, transactionalId);
- Object transactionManager = getTransactionManager();
- synchronized (transactionManager) {
- setField(transactionManager, "transactionalId", transactionalId);
- setField(
- transactionManager,
- "currentState",
- getTransactionManagerState("UNINITIALIZED"));
- this.transactionalId = transactionalId;
- }
- }
- }
-
- /**
- * Besides committing {@link org.apache.kafka.clients.producer.KafkaProducer#commitTransaction}
- * is also adding new partitions to the transaction. flushNewPartitions method is moving this
- * logic to pre-commit/flush, to make resumeTransaction simpler. Otherwise resumeTransaction
- * would require to restore state of the not yet added/"in-flight" partitions.
- */
- private void flushNewPartitions() {
- LOG.info("Flushing new partitions");
- TransactionalRequestResult result = enqueueNewPartitions();
- Object sender = getField("sender");
- invoke(sender, "wakeup");
- result.await();
- }
-
- /**
- * Enqueues new transactions at the transaction manager and returns a {@link
- * TransactionalRequestResult} that allows waiting on them.
- *
- * <p>If there are no new transactions we return a {@link TransactionalRequestResult} that is
- * already done.
- */
- private TransactionalRequestResult enqueueNewPartitions() {
- Object transactionManager = getTransactionManager();
- synchronized (transactionManager) {
- Object newPartitionsInTransaction =
- getField(transactionManager, "newPartitionsInTransaction");
- Object newPartitionsInTransactionIsEmpty =
- invoke(newPartitionsInTransaction, "isEmpty");
- TransactionalRequestResult result;
- if (newPartitionsInTransactionIsEmpty instanceof Boolean
- && !((Boolean) newPartitionsInTransactionIsEmpty)) {
- Object txnRequestHandler =
- invoke(transactionManager, "addPartitionsToTransactionHandler");
- invoke(
- transactionManager,
- "enqueueRequest",
- new Class[] {txnRequestHandler.getClass().getSuperclass()},
- new Object[] {txnRequestHandler});
- result =
- (TransactionalRequestResult)
- getField(
- txnRequestHandler,
- txnRequestHandler.getClass().getSuperclass(),
- "result");
- } else {
- // we don't have an operation but this operation string is also used in
- // addPartitionsToTransactionHandler.
- result = new TransactionalRequestResult("AddPartitionsToTxn");
- result.done();
- }
- return result;
- }
- }
-
- private static Object invoke(Object object, String methodName, Object... args) {
- Class<?>[] argTypes = new Class[args.length];
- for (int i = 0; i < args.length; i++) {
- argTypes[i] = args[i].getClass();
- }
- return invoke(object, methodName, argTypes, args);
- }
-
- private static Object invoke(
- Object object, String methodName, Class<?>[] argTypes, Object[] args) {
- try {
- Method method = object.getClass().getDeclaredMethod(methodName, argTypes);
- method.setAccessible(true);
- return method.invoke(object, args);
- } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
- throw new RuntimeException("Incompatible KafkaProducer version", e);
- }
- }
-
- private Object getField(String fieldName) {
- return getField(this, KafkaProducer.class, fieldName);
- }
-
- /**
- * Gets and returns the field {@code fieldName} from the given Object {@code object} using
- * reflection.
- */
- private static Object getField(Object object, String fieldName) {
- return getField(object, object.getClass(), fieldName);
- }
-
- /**
- * Gets and returns the field {@code fieldName} from the given Object {@code object} using
- * reflection.
- */
- private static Object getField(Object object, Class<?> clazz, String fieldName) {
- try {
- Field field = clazz.getDeclaredField(fieldName);
- field.setAccessible(true);
- return field.get(object);
- } catch (NoSuchFieldException | IllegalAccessException e) {
- throw new RuntimeException("Incompatible KafkaProducer version", e);
- }
- }
-
- /**
- * Instead of obtaining producerId and epoch from the transaction coordinator, re-use previously
- * obtained ones, so that we can resume transaction after a restart. Implementation of this
- * method is based on {@link KafkaProducer#initTransactions}.
- * https://github.com/apache/kafka/commit/5d2422258cb975a137a42a4e08f03573c49a387e#diff-f4ef1afd8792cd2a2e9069cd7ddea630
- */
- public void resumeTransaction(long producerId, short epoch) {
- checkState(!inTransaction, "Already in transaction %s", transactionalId);
- checkState(
- producerId >= 0 && epoch >= 0,
- "Incorrect values for producerId %s and epoch %s",
- producerId,
- epoch);
- LOG.info(
- "Attempting to resume transaction {} with producerId {} and epoch {}",
- transactionalId,
- producerId,
- epoch);
-
- Object transactionManager = getTransactionManager();
- synchronized (transactionManager) {
- Object topicPartitionBookkeeper =
- getField(transactionManager, "topicPartitionBookkeeper");
-
- transitionTransactionManagerStateTo(transactionManager, "INITIALIZING");
- invoke(topicPartitionBookkeeper, "reset");
-
- setField(
- transactionManager,
- PRODUCER_ID_AND_EPOCH_FIELD_NAME,
- createProducerIdAndEpoch(producerId, epoch));
-
- transitionTransactionManagerStateTo(transactionManager, "READY");
-
- transitionTransactionManagerStateTo(transactionManager, "IN_TRANSACTION");
- setField(transactionManager, "transactionStarted", true);
- this.inTransaction = true;
- }
- }
-
- private static Object createProducerIdAndEpoch(long producerId, short epoch) {
- try {
- Field field =
- TransactionManager.class.getDeclaredField(PRODUCER_ID_AND_EPOCH_FIELD_NAME);
- Class<?> clazz = field.getType();
- Constructor<?> constructor = clazz.getDeclaredConstructor(Long.TYPE, Short.TYPE);
- constructor.setAccessible(true);
- return constructor.newInstance(producerId, epoch);
- } catch (InvocationTargetException
- | InstantiationException
- | IllegalAccessException
- | NoSuchFieldException
- | NoSuchMethodException e) {
- throw new RuntimeException("Incompatible KafkaProducer version", e);
- }
- }
-
- /**
- * Sets the field {@code fieldName} on the given Object {@code object} to {@code value} using
- * reflection.
- */
- private static void setField(Object object, String fieldName, Object value) {
- setField(object, object.getClass(), fieldName, value);
- }
-
- private static void setField(Object object, Class<?> clazz, String fieldName, Object value) {
- try {
- Field field = clazz.getDeclaredField(fieldName);
- field.setAccessible(true);
- field.set(object, value);
- } catch (NoSuchFieldException | IllegalAccessException e) {
- throw new RuntimeException("Incompatible KafkaProducer version", e);
- }
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- private static Enum<?> getTransactionManagerState(String enumName) {
- try {
- Class<Enum> cl = (Class<Enum>) Class.forName(TRANSACTION_MANAGER_STATE_ENUM);
- return Enum.valueOf(cl, enumName);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("Incompatible KafkaProducer version", e);
- }
- }
-
- private Object getTransactionManager() {
- return getField(TRANSACTION_MANAGER_FIELD_NAME);
- }
-
- private static void transitionTransactionManagerStateTo(
- Object transactionManager, String state) {
- invoke(transactionManager, "transitionTo", getTransactionManagerState(state));
- }
-
- @Override
- public String toString() {
- return "FlinkKafkaInternalProducer{"
- + "transactionalId='"
- + transactionalId
- + "', inTransaction="
- + inTransaction
- + ", closed="
- + closed
- + '}';
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittable.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittable.java
deleted file mode 100644
index 72a3281e683..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittable.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import javax.annotation.Nullable;
-
-import java.util.Objects;
-import java.util.Optional;
-import java.util.function.Consumer;
-
-/**
- * This class holds the necessary information to construct a new {@link FlinkKafkaInternalProducer}
- * to commit transactions in {@link KafkaCommitter}.
- */
-class KafkaCommittable {
-
- private final long producerId;
- private final short epoch;
- private final String transactionalId;
- @Nullable private Recyclable<? extends FlinkKafkaInternalProducer<?, ?>> producer;
-
- public KafkaCommittable(
- long producerId,
- short epoch,
- String transactionalId,
- @Nullable Recyclable<? extends FlinkKafkaInternalProducer<?, ?>> producer) {
- this.producerId = producerId;
- this.epoch = epoch;
- this.transactionalId = transactionalId;
- this.producer = producer;
- }
-
- public static <K, V> KafkaCommittable of(
- FlinkKafkaInternalProducer<K, V> producer,
- Consumer<FlinkKafkaInternalProducer<K, V>> recycler) {
- return new KafkaCommittable(
- producer.getProducerId(),
- producer.getEpoch(),
- producer.getTransactionalId(),
- new Recyclable<>(producer, recycler));
- }
-
- public long getProducerId() {
- return producerId;
- }
-
- public short getEpoch() {
- return epoch;
- }
-
- public String getTransactionalId() {
- return transactionalId;
- }
-
- public Optional<Recyclable<? extends FlinkKafkaInternalProducer<?, ?>>> getProducer() {
- return Optional.ofNullable(producer);
- }
-
- @Override
- public String toString() {
- return "KafkaCommittable{"
- + "producerId="
- + producerId
- + ", epoch="
- + epoch
- + ", transactionalId="
- + transactionalId
- + '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- KafkaCommittable that = (KafkaCommittable) o;
- return producerId == that.producerId
- && epoch == that.epoch
- && transactionalId.equals(that.transactionalId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(producerId, epoch, transactionalId);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittableSerializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittableSerializer.java
deleted file mode 100644
index 78f1472c648..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommittableSerializer.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-class KafkaCommittableSerializer implements SimpleVersionedSerializer<KafkaCommittable> {
-
- @Override
- public int getVersion() {
- return 1;
- }
-
- @Override
- public byte[] serialize(KafkaCommittable state) throws IOException {
- try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final DataOutputStream out = new DataOutputStream(baos)) {
- out.writeShort(state.getEpoch());
- out.writeLong(state.getProducerId());
- out.writeUTF(state.getTransactionalId());
- out.flush();
- return baos.toByteArray();
- }
- }
-
- @Override
- public KafkaCommittable deserialize(int version, byte[] serialized) throws IOException {
- try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- final DataInputStream in = new DataInputStream(bais)) {
- final short epoch = in.readShort();
- final long producerId = in.readLong();
- final String transactionalId = in.readUTF();
- return new KafkaCommittable(producerId, epoch, transactionalId, null);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommitter.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommitter.java
deleted file mode 100644
index 4dbeaf9e715..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaCommitter.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.api.connector.sink2.Committer;
-
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.errors.InvalidTxnStateException;
-import org.apache.kafka.common.errors.ProducerFencedException;
-import org.apache.kafka.common.errors.RetriableException;
-import org.apache.kafka.common.errors.UnknownProducerIdException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Optional;
-import java.util.Properties;
-
-/**
- * Committer implementation for {@link KafkaSink}
- *
- * <p>The committer is responsible to finalize the Kafka transactions by committing them.
- */
-class KafkaCommitter implements Committer<KafkaCommittable>, Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaCommitter.class);
- public static final String UNKNOWN_PRODUCER_ID_ERROR_MESSAGE =
- "because of a bug in the Kafka broker (KAFKA-9310). Please upgrade to Kafka 2.5+. If you are running with concurrent checkpoints, you also may want to try without them.\n"
- + "To avoid data loss, the application will restart.";
-
- private final Properties kafkaProducerConfig;
-
- @Nullable private FlinkKafkaInternalProducer<?, ?> recoveryProducer;
-
- KafkaCommitter(Properties kafkaProducerConfig) {
- this.kafkaProducerConfig = kafkaProducerConfig;
- }
-
- @Override
- public void commit(Collection<CommitRequest<KafkaCommittable>> requests)
- throws IOException, InterruptedException {
- for (CommitRequest<KafkaCommittable> request : requests) {
- final KafkaCommittable committable = request.getCommittable();
- final String transactionalId = committable.getTransactionalId();
- LOG.debug("Committing Kafka transaction {}", transactionalId);
- Optional<Recyclable<? extends FlinkKafkaInternalProducer<?, ?>>> recyclable =
- committable.getProducer();
- FlinkKafkaInternalProducer<?, ?> producer;
- try {
- producer =
- recyclable
- .<FlinkKafkaInternalProducer<?, ?>>map(Recyclable::getObject)
- .orElseGet(() -> getRecoveryProducer(committable));
- producer.commitTransaction();
- producer.flush();
- recyclable.ifPresent(Recyclable::close);
- } catch (RetriableException e) {
- LOG.warn(
- "Encountered retriable exception while committing {}.", transactionalId, e);
- request.retryLater();
- } catch (ProducerFencedException e) {
- // initTransaction has been called on this transaction before
- LOG.error(
- "Unable to commit transaction ({}) because its producer is already fenced."
- + " This means that you either have a different producer with the same '{}' (this is"
- + " unlikely with the '{}' as all generated ids are unique and shouldn't be reused)"
- + " or recovery took longer than '{}' ({}ms). In both cases this most likely signals data loss,"
- + " please consult the Flink documentation for more details.",
- request,
- ProducerConfig.TRANSACTIONAL_ID_CONFIG,
- KafkaSink.class.getSimpleName(),
- ProducerConfig.TRANSACTION_TIMEOUT_CONFIG,
- kafkaProducerConfig.getProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG),
- e);
- recyclable.ifPresent(Recyclable::close);
- request.signalFailedWithKnownReason(e);
- } catch (InvalidTxnStateException e) {
- // This exception only occurs when aborting after a commit or vice versa.
- // It does not appear on double commits or double aborts.
- LOG.error(
- "Unable to commit transaction ({}) because it's in an invalid state. "
- + "Most likely the transaction has been aborted for some reason. Please check the Kafka logs for more details.",
- request,
- e);
- recyclable.ifPresent(Recyclable::close);
- request.signalFailedWithKnownReason(e);
- } catch (UnknownProducerIdException e) {
- LOG.error(
- "Unable to commit transaction ({}) " + UNKNOWN_PRODUCER_ID_ERROR_MESSAGE,
- request,
- e);
- recyclable.ifPresent(Recyclable::close);
- request.signalFailedWithKnownReason(e);
- } catch (Exception e) {
- LOG.error(
- "Transaction ({}) encountered error and data has been potentially lost.",
- request,
- e);
- recyclable.ifPresent(Recyclable::close);
- request.signalFailedWithUnknownReason(e);
- }
- }
- }
-
- @Override
- public void close() {
- if (recoveryProducer != null) {
- recoveryProducer.close();
- }
- }
-
- /**
- * Creates a producer that can commit into the same transaction as the upstream producer that
- * was serialized into {@link KafkaCommittable}.
- */
- private FlinkKafkaInternalProducer<?, ?> getRecoveryProducer(KafkaCommittable committable) {
- if (recoveryProducer == null) {
- recoveryProducer =
- new FlinkKafkaInternalProducer<>(
- kafkaProducerConfig, committable.getTransactionalId());
- } else {
- recoveryProducer.setTransactionId(committable.getTransactionalId());
- }
- recoveryProducer.resumeTransaction(committable.getProducerId(), committable.getEpoch());
- return recoveryProducer;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchema.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchema.java
deleted file mode 100644
index 9d081c755a1..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchema.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-
-import org.apache.kafka.clients.producer.ProducerRecord;
-
-import javax.annotation.Nullable;
-
-import java.io.Serializable;
-
-/**
- * A serialization schema which defines how to convert a value of type {@code T} to {@link
- * ProducerRecord}.
- *
- * @param <T> the type of values being serialized
- */
-@PublicEvolving
-public interface KafkaRecordSerializationSchema<T> extends Serializable {
-
- /**
- * Initialization method for the schema. It is called before the actual working methods {@link
- * #serialize(Object, KafkaSinkContext, Long)} and thus suitable for one time setup work.
- *
- * <p>The provided {@link SerializationSchema.InitializationContext} can be used to access
- * additional features such as e.g. registering user metrics.
- *
- * @param context Contextual information that can be used during initialization.
- * @param sinkContext runtime information i.e. partitions, subtaskId
- */
- default void open(
- SerializationSchema.InitializationContext context, KafkaSinkContext sinkContext)
- throws Exception {}
-
- /**
- * Serializes given element and returns it as a {@link ProducerRecord}.
- *
- * @param element element to be serialized
- * @param context context to possibly determine target partition
- * @param timestamp timestamp
- * @return Kafka {@link ProducerRecord} or null if the given element cannot be serialized
- */
- @Nullable
- ProducerRecord<byte[], byte[]> serialize(T element, KafkaSinkContext context, Long timestamp);
-
- /** Context providing information of the kafka record target location. */
- @Internal
- interface KafkaSinkContext {
-
- /**
- * Get the ID of the subtask the KafkaSink is running on. The numbering starts from 0 and
- * goes up to parallelism-1. (parallelism as returned by {@link
- * #getNumberOfParallelInstances()}
- *
- * @return ID of subtask
- */
- int getParallelInstanceId();
-
- /** @return number of parallel KafkaSink tasks. */
- int getNumberOfParallelInstances();
-
- /**
- * For a given topic id retrieve the available partitions.
- *
- * <p>After the first retrieval the returned partitions are cached. If the partitions are
- * updated the job has to be restarted to make the change visible.
- *
- * @param topic kafka topic with partitions
- * @return the ids of the currently available partitions
- */
- int[] getPartitionsForTopic(String topic);
- }
-
- /**
- * Creates a default schema builder to provide common building blocks i.e. key serialization,
- * value serialization, partitioning.
- *
- * @param <T> type of incoming elements
- * @return {@link KafkaRecordSerializationSchemaBuilder}
- */
- static <T> KafkaRecordSerializationSchemaBuilder<T> builder() {
- return new KafkaRecordSerializationSchemaBuilder<>();
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchemaBuilder.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchemaBuilder.java
deleted file mode 100644
index 59864a37323..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaRecordSerializationSchemaBuilder.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner;
-
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.common.serialization.Serializer;
-
-import javax.annotation.Nullable;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.OptionalInt;
-import java.util.function.Function;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * Builder to construct {@link KafkaRecordSerializationSchema}.
- *
- * <p>This class should give a first entrypoint when trying to serialize elements to {@link
- * ProducerRecord}. The following examples show some of the possibilities.
- *
- * <pre>Simple key-value serialization:
- * {@code
- * KafkaRecordSerializationSchema.builder()
- * .setTopic("topic)
- * .setKeySerializationSchema(new SimpleStringSchema())
- * .setValueSerializationSchema(new SimpleStringSchema())
- * .build()
- * }</pre>
- *
- * <pre>Using Kafka's serialization stack:
- * {@code
- * KafkaRecordSerializationSchema.builder()
- * .setTopic("topic)
- * .setKeySerializer(StringSerializer.class)
- * .setKafkaValueSerializer(StringSerializer.class)
- * .build()
- * }</pre>
- *
- * <pre>With custom partitioner:
- * {@code
- * KafkaRecordSerializationSchema.builder()
- * .setTopic("topic)
- * .setPartitioner(MY_FLINK_PARTITIONER)
- * .setValueSerializationSchema(StringSerializer.class)
- * .build()
- * }</pre>
- *
- * <p>The different serialization methods for key and value are mutually exclusive thus i.e. it is
- * not possible to use {@link #setKeySerializationSchema(SerializationSchema)} and {@link
- * #setKafkaKeySerializer(Class)} on the same builder instance.
- *
- * <p>It is necessary to configure exactly one serialization method for the value and a topic.
- *
- * @param <IN> type of records to be serialized
- * @see KafkaRecordSerializationSchema#builder()
- */
-@PublicEvolving
-public class KafkaRecordSerializationSchemaBuilder<IN> {
-
- @Nullable private Function<? super IN, String> topicSelector;
- @Nullable private SerializationSchema<? super IN> valueSerializationSchema;
- @Nullable private FlinkKafkaPartitioner<? super IN> partitioner;
- @Nullable private SerializationSchema<? super IN> keySerializationSchema;
-
- /**
- * Sets a custom partitioner determining the target partition of the target topic.
- *
- * @param partitioner
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setPartitioner(
- FlinkKafkaPartitioner<? super T> partitioner) {
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.partitioner = checkNotNull(partitioner);
- return self;
- }
-
- /**
- * Sets a fixed topic which used as destination for all records.
- *
- * @param topic
- * @return {@code this}
- */
- public KafkaRecordSerializationSchemaBuilder<IN> setTopic(String topic) {
- checkState(this.topicSelector == null, "Topic selector already set.");
- checkNotNull(topic);
- this.topicSelector = new CachingTopicSelector<>((e) -> topic);
- return this;
- }
-
- /**
- * Sets a topic selector which computes the target topic for every incoming record.
- *
- * @param topicSelector
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setTopicSelector(
- TopicSelector<? super T> topicSelector) {
- checkState(this.topicSelector == null, "Topic selector already set.");
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.topicSelector = new CachingTopicSelector<>(checkNotNull(topicSelector));
- return self;
- }
-
- /**
- * Sets a {@link SerializationSchema} which is used to serialize the incoming element to the key
- * of the {@link ProducerRecord}.
- *
- * @param keySerializationSchema
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setKeySerializationSchema(
- SerializationSchema<? super T> keySerializationSchema) {
- checkKeySerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.keySerializationSchema = checkNotNull(keySerializationSchema);
- return self;
- }
-
- /**
- * Sets Kafka's {@link Serializer} to serialize incoming elements to the key of the {@link
- * ProducerRecord}.
- *
- * @param keySerializer
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setKafkaKeySerializer(
- Class<? extends Serializer<? super T>> keySerializer) {
- checkKeySerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.keySerializationSchema =
- new KafkaSerializerWrapper<>(keySerializer, true, topicSelector);
- return self;
- }
-
- /**
- * Sets a configurable Kafka {@link Serializer} and pass a configuration to serialize incoming
- * elements to the key of the {@link ProducerRecord}.
- *
- * @param keySerializer
- * @param configuration
- * @param <S> type of the used serializer class
- * @return {@code this}
- */
- public <T extends IN, S extends Serializer<? super T>>
- KafkaRecordSerializationSchemaBuilder<T> setKafkaKeySerializer(
- Class<S> keySerializer, Map<String, String> configuration) {
- checkKeySerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.keySerializationSchema =
- new KafkaSerializerWrapper<>(keySerializer, true, configuration, topicSelector);
- return self;
- }
-
- /**
- * Sets a {@link SerializationSchema} which is used to serialize the incoming element to the
- * value of the {@link ProducerRecord}.
- *
- * @param valueSerializationSchema
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setValueSerializationSchema(
- SerializationSchema<T> valueSerializationSchema) {
- checkValueSerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.valueSerializationSchema = checkNotNull(valueSerializationSchema);
- return self;
- }
-
- @SuppressWarnings("unchecked")
- private <T extends IN> KafkaRecordSerializationSchemaBuilder<T> self() {
- return (KafkaRecordSerializationSchemaBuilder<T>) this;
- }
-
- /**
- * Sets Kafka's {@link Serializer} to serialize incoming elements to the value of the {@link
- * ProducerRecord}.
- *
- * @param valueSerializer
- * @return {@code this}
- */
- public <T extends IN> KafkaRecordSerializationSchemaBuilder<T> setKafkaValueSerializer(
- Class<? extends Serializer<? super T>> valueSerializer) {
- checkValueSerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.valueSerializationSchema =
- new KafkaSerializerWrapper<>(valueSerializer, false, topicSelector);
- return self;
- }
-
- /**
- * Sets a configurable Kafka {@link Serializer} and pass a configuration to serialize incoming
- * elements to the value of the {@link ProducerRecord}.
- *
- * @param valueSerializer
- * @param configuration
- * @param <S> type of the used serializer class
- * @return {@code this}
- */
- public <T extends IN, S extends Serializer<? super T>>
- KafkaRecordSerializationSchemaBuilder<T> setKafkaValueSerializer(
- Class<S> valueSerializer, Map<String, String> configuration) {
- checkValueSerializerNotSet();
- KafkaRecordSerializationSchemaBuilder<T> self = self();
- self.valueSerializationSchema =
- new KafkaSerializerWrapper<>(valueSerializer, false, configuration, topicSelector);
- return self;
- }
-
- /**
- * Constructs the {@link KafkaRecordSerializationSchemaBuilder} with the configured properties.
- *
- * @return {@link KafkaRecordSerializationSchema}
- */
- public KafkaRecordSerializationSchema<IN> build() {
- checkState(valueSerializationSchema != null, "No value serializer is configured.");
- checkState(topicSelector != null, "No topic selector is configured.");
- return new KafkaRecordSerializationSchemaWrapper<>(
- topicSelector, valueSerializationSchema, keySerializationSchema, partitioner);
- }
-
- private void checkValueSerializerNotSet() {
- checkState(valueSerializationSchema == null, "Value serializer already set.");
- }
-
- private void checkKeySerializerNotSet() {
- checkState(keySerializationSchema == null, "Key serializer already set.");
- }
-
- private static class CachingTopicSelector<IN> implements Function<IN, String>, Serializable {
-
- private static final int CACHE_RESET_SIZE = 5;
- private final Map<IN, String> cache;
- private final TopicSelector<IN> topicSelector;
-
- CachingTopicSelector(TopicSelector<IN> topicSelector) {
- this.topicSelector = topicSelector;
- this.cache = new HashMap<>();
- }
-
- @Override
- public String apply(IN in) {
- final String topic = cache.getOrDefault(in, topicSelector.apply(in));
- cache.put(in, topic);
- if (cache.size() == CACHE_RESET_SIZE) {
- cache.clear();
- }
- return topic;
- }
- }
-
- private static class KafkaRecordSerializationSchemaWrapper<IN>
- implements KafkaRecordSerializationSchema<IN> {
- private final SerializationSchema<? super IN> valueSerializationSchema;
- private final Function<? super IN, String> topicSelector;
- private final FlinkKafkaPartitioner<? super IN> partitioner;
- private final SerializationSchema<? super IN> keySerializationSchema;
-
- KafkaRecordSerializationSchemaWrapper(
- Function<? super IN, String> topicSelector,
- SerializationSchema<? super IN> valueSerializationSchema,
- @Nullable SerializationSchema<? super IN> keySerializationSchema,
- @Nullable FlinkKafkaPartitioner<? super IN> partitioner) {
- this.topicSelector = checkNotNull(topicSelector);
- this.valueSerializationSchema = checkNotNull(valueSerializationSchema);
- this.partitioner = partitioner;
- this.keySerializationSchema = keySerializationSchema;
- }
-
- @Override
- public void open(
- SerializationSchema.InitializationContext context, KafkaSinkContext sinkContext)
- throws Exception {
- valueSerializationSchema.open(context);
- if (keySerializationSchema != null) {
- keySerializationSchema.open(context);
- }
- if (partitioner != null) {
- partitioner.open(
- sinkContext.getParallelInstanceId(),
- sinkContext.getNumberOfParallelInstances());
- }
- }
-
- @Override
- public ProducerRecord<byte[], byte[]> serialize(
- IN element, KafkaSinkContext context, Long timestamp) {
- final String targetTopic = topicSelector.apply(element);
- final byte[] value = valueSerializationSchema.serialize(element);
- byte[] key = null;
- if (keySerializationSchema != null) {
- key = keySerializationSchema.serialize(element);
- }
- final OptionalInt partition =
- partitioner != null
- ? OptionalInt.of(
- partitioner.partition(
- element,
- key,
- value,
- targetTopic,
- context.getPartitionsForTopic(targetTopic)))
- : OptionalInt.empty();
-
- return new ProducerRecord<>(
- targetTopic,
- partition.isPresent() ? partition.getAsInt() : null,
- timestamp == null || timestamp < 0L ? null : timestamp,
- key,
- value);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSerializerWrapper.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSerializerWrapper.java
deleted file mode 100644
index f2120568b0f..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSerializerWrapper.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.TemporaryClassLoaderContext;
-
-import org.apache.kafka.common.Configurable;
-import org.apache.kafka.common.serialization.Serializer;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.function.Function;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/** Wrapper for Kafka {@link Serializer}. */
-class KafkaSerializerWrapper<IN> implements SerializationSchema<IN> {
-
- private final Class<? extends Serializer<? super IN>> serializerClass;
- // Whether the serializer is for key or value.
- private final boolean isKey;
- private final Map<String, String> config;
- private final Function<? super IN, String> topicSelector;
-
- private transient Serializer<? super IN> serializer;
-
- KafkaSerializerWrapper(
- Class<? extends Serializer<? super IN>> serializerClass,
- boolean isKey,
- Map<String, String> config,
- Function<? super IN, String> topicSelector) {
- this.serializerClass = checkNotNull(serializerClass);
- this.isKey = isKey;
- this.config = checkNotNull(config);
- this.topicSelector = checkNotNull(topicSelector);
- }
-
- KafkaSerializerWrapper(
- Class<? extends Serializer<? super IN>> serializerClass,
- boolean isKey,
- Function<? super IN, String> topicSelector) {
- this(serializerClass, isKey, Collections.emptyMap(), topicSelector);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(InitializationContext context) throws Exception {
- final ClassLoader userCodeClassLoader = context.getUserCodeClassLoader().asClassLoader();
- try (TemporaryClassLoaderContext ignored =
- TemporaryClassLoaderContext.of(userCodeClassLoader)) {
- serializer =
- InstantiationUtil.instantiate(
- serializerClass.getName(),
- Serializer.class,
- getClass().getClassLoader());
-
- if (serializer instanceof Configurable) {
- ((Configurable) serializer).configure(config);
- } else {
- serializer.configure(config, isKey);
- }
- } catch (Exception e) {
- throw new IOException("Failed to instantiate the serializer of class " + serializer, e);
- }
- }
-
- @Override
- public byte[] serialize(IN element) {
- checkState(serializer != null, "Call open() once before trying to serialize elements.");
- return serializer.serialize(topicSelector.apply(element), element);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSink.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSink.java
deleted file mode 100644
index 6f74aaed5fa..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSink.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.connector.sink2.Committer;
-import org.apache.flink.api.connector.sink2.StatefulSink;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Properties;
-
-/**
- * Flink Sink to produce data into a Kafka topic. The sink supports all delivery guarantees
- * described by {@link DeliveryGuarantee}.
- * <li>{@link DeliveryGuarantee#NONE} does not provide any guarantees: messages may be lost in case
- * of issues on the Kafka broker and messages may be duplicated in case of a Flink failure.
- * <li>{@link DeliveryGuarantee#AT_LEAST_ONCE} the sink will wait for all outstanding records in the
- * Kafka buffers to be acknowledged by the Kafka producer on a checkpoint. No messages will be
- * lost in case of any issue with the Kafka brokers but messages may be duplicated when Flink
- * restarts.
- * <li>{@link DeliveryGuarantee#EXACTLY_ONCE}: In this mode the KafkaSink will write all messages in
- * a Kafka transaction that will be committed to Kafka on a checkpoint. Thus, if the consumer
- * reads only committed data (see Kafka consumer config isolation.level), no duplicates will be
- * seen in case of a Flink restart. However, this delays record writing effectively until a
- * checkpoint is written, so adjust the checkpoint duration accordingly. Please ensure that you
- * use unique {@link #transactionalIdPrefix}s across your applications running on the same Kafka
- * cluster such that multiple running jobs do not interfere in their transactions! Additionally,
- * it is highly recommended to tweak Kafka transaction timeout (link) >> maximum checkpoint
- * duration + maximum restart duration or data loss may happen when Kafka expires an uncommitted
- * transaction.
- *
- * @param <IN> type of the records written to Kafka
- * @see KafkaSinkBuilder on how to construct a KafkaSink
- */
-@PublicEvolving
-public class KafkaSink<IN>
- implements StatefulSink<IN, KafkaWriterState>,
- TwoPhaseCommittingSink<IN, KafkaCommittable> {
-
- private final DeliveryGuarantee deliveryGuarantee;
-
- private final KafkaRecordSerializationSchema<IN> recordSerializer;
- private final Properties kafkaProducerConfig;
- private final String transactionalIdPrefix;
-
- KafkaSink(
- DeliveryGuarantee deliveryGuarantee,
- Properties kafkaProducerConfig,
- String transactionalIdPrefix,
- KafkaRecordSerializationSchema<IN> recordSerializer) {
- this.deliveryGuarantee = deliveryGuarantee;
- this.kafkaProducerConfig = kafkaProducerConfig;
- this.transactionalIdPrefix = transactionalIdPrefix;
- this.recordSerializer = recordSerializer;
- }
-
- /**
- * Create a {@link KafkaSinkBuilder} to construct a new {@link KafkaSink}.
- *
- * @param <IN> type of incoming records
- * @return {@link KafkaSinkBuilder}
- */
- public static <IN> KafkaSinkBuilder<IN> builder() {
- return new KafkaSinkBuilder<>();
- }
-
- @Internal
- @Override
- public Committer<KafkaCommittable> createCommitter() throws IOException {
- return new KafkaCommitter(kafkaProducerConfig);
- }
-
- @Internal
- @Override
- public SimpleVersionedSerializer<KafkaCommittable> getCommittableSerializer() {
- return new KafkaCommittableSerializer();
- }
-
- @Internal
- @Override
- public KafkaWriter<IN> createWriter(InitContext context) throws IOException {
- return new KafkaWriter<IN>(
- deliveryGuarantee,
- kafkaProducerConfig,
- transactionalIdPrefix,
- context,
- recordSerializer,
- context.asSerializationSchemaInitializationContext(),
- Collections.emptyList());
- }
-
- @Internal
- @Override
- public KafkaWriter<IN> restoreWriter(
- InitContext context, Collection<KafkaWriterState> recoveredState) throws IOException {
- return new KafkaWriter<>(
- deliveryGuarantee,
- kafkaProducerConfig,
- transactionalIdPrefix,
- context,
- recordSerializer,
- context.asSerializationSchemaInitializationContext(),
- recoveredState);
- }
-
- @Internal
- @Override
- public SimpleVersionedSerializer<KafkaWriterState> getWriterStateSerializer() {
- return new KafkaWriterStateSerializer();
- }
-
- @VisibleForTesting
- protected Properties getKafkaProducerConfig() {
- return kafkaProducerConfig;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSinkBuilder.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSinkBuilder.java
deleted file mode 100644
index f0c20cfc01a..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaSinkBuilder.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Properties;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * Builder to construct {@link KafkaSink}.
- *
- * <p>The following example shows the minimum setup to create a KafkaSink that writes String values
- * to a Kafka topic.
- *
- * <pre>{@code
- * KafkaSink<String> sink = KafkaSink
- * .<String>builder
- * .setBootstrapServers(MY_BOOTSTRAP_SERVERS)
- * .setRecordSerializer(MY_RECORD_SERIALIZER)
- * .build();
- * }</pre>
- *
- * <p>One can also configure different {@link DeliveryGuarantee} by using {@link
- * #setDeliveryGuarantee(DeliveryGuarantee)} but keep in mind when using {@link
- * DeliveryGuarantee#EXACTLY_ONCE} one must set the transactionalIdPrefix {@link
- * #setTransactionalIdPrefix(String)}.
- *
- * @see KafkaSink for a more detailed explanation of the different guarantees.
- * @param <IN> type of the records written to Kafka
- */
-@PublicEvolving
-public class KafkaSinkBuilder<IN> {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkBuilder.class);
- private static final Duration DEFAULT_KAFKA_TRANSACTION_TIMEOUT = Duration.ofHours(1);
- private static final String[] warnKeys =
- new String[] {
- ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
- ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG
- };
- private static final int MAXIMUM_PREFIX_BYTES = 64000;
-
- private DeliveryGuarantee deliveryGuarantee = DeliveryGuarantee.NONE;
- private String transactionalIdPrefix = "kafka-sink";
-
- private final Properties kafkaProducerConfig;
- private KafkaRecordSerializationSchema<IN> recordSerializer;
-
- KafkaSinkBuilder() {
- kafkaProducerConfig = new Properties();
- kafkaProducerConfig.put(
- ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
- kafkaProducerConfig.put(
- ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
- kafkaProducerConfig.put(
- ProducerConfig.TRANSACTION_TIMEOUT_CONFIG,
- (int) DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMillis());
- }
-
- /**
- * Sets the wanted the {@link DeliveryGuarantee}. The default delivery guarantee is {@link
- * #deliveryGuarantee}.
- *
- * @param deliveryGuarantee
- * @return {@link KafkaSinkBuilder}
- */
- public KafkaSinkBuilder<IN> setDeliveryGuarantee(DeliveryGuarantee deliveryGuarantee) {
- this.deliveryGuarantee = checkNotNull(deliveryGuarantee, "deliveryGuarantee");
- return this;
- }
-
- /**
- * Sets the wanted the {@link DeliveryGuarantee}. The default delivery guarantee is {@link
- * #deliveryGuarantee}.
- *
- * @param deliveryGuarantee
- * @return {@link KafkaSinkBuilder}
- * @deprecated Will be removed in future versions. Use {@link #setDeliveryGuarantee} instead.
- */
- @Deprecated
- public KafkaSinkBuilder<IN> setDeliverGuarantee(DeliveryGuarantee deliveryGuarantee) {
- this.deliveryGuarantee = checkNotNull(deliveryGuarantee, "deliveryGuarantee");
- return this;
- }
-
- /**
- * Sets the configuration which used to instantiate all used {@link
- * org.apache.kafka.clients.producer.KafkaProducer}.
- *
- * @param props
- * @return {@link KafkaSinkBuilder}
- */
- public KafkaSinkBuilder<IN> setKafkaProducerConfig(Properties props) {
- checkNotNull(props);
- Arrays.stream(warnKeys)
- .filter(props::containsKey)
- .forEach(k -> LOG.warn("Overwriting the '{}' is not recommended", k));
-
- kafkaProducerConfig.putAll(props);
- return this;
- }
-
- public KafkaSinkBuilder<IN> setProperty(String key, String value) {
- checkNotNull(key);
- Arrays.stream(warnKeys)
- .filter(key::equals)
- .forEach(k -> LOG.warn("Overwriting the '{}' is not recommended", k));
-
- kafkaProducerConfig.setProperty(key, value);
- return this;
- }
-
- /**
- * Sets the {@link KafkaRecordSerializationSchema} that transforms incoming records to {@link
- * org.apache.kafka.clients.producer.ProducerRecord}s.
- *
- * @param recordSerializer
- * @return {@link KafkaSinkBuilder}
- */
- public KafkaSinkBuilder<IN> setRecordSerializer(
- KafkaRecordSerializationSchema<IN> recordSerializer) {
- this.recordSerializer = checkNotNull(recordSerializer, "recordSerializer");
- ClosureCleaner.clean(
- this.recordSerializer, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
- return this;
- }
-
- /**
- * Sets the prefix for all created transactionalIds if {@link DeliveryGuarantee#EXACTLY_ONCE} is
- * configured.
- *
- * <p>It is mandatory to always set this value with {@link DeliveryGuarantee#EXACTLY_ONCE} to
- * prevent corrupted transactions if multiple jobs using the KafkaSink run against the same
- * Kafka Cluster. The default prefix is {@link #transactionalIdPrefix}.
- *
- * <p>The size of the prefix is capped by {@link #MAXIMUM_PREFIX_BYTES} formatted with UTF-8.
- *
- * <p>It is important to keep the prefix stable across application restarts. If the prefix
- * changes it might happen that lingering transactions are not correctly aborted and newly
- * written messages are not immediately consumable until the transactions timeout.
- *
- * @param transactionalIdPrefix
- * @return {@link KafkaSinkBuilder}
- */
- public KafkaSinkBuilder<IN> setTransactionalIdPrefix(String transactionalIdPrefix) {
- this.transactionalIdPrefix = checkNotNull(transactionalIdPrefix, "transactionalIdPrefix");
- checkState(
- transactionalIdPrefix.getBytes(StandardCharsets.UTF_8).length
- <= MAXIMUM_PREFIX_BYTES,
- "The configured prefix is too long and the resulting transactionalId might exceed Kafka's transactionalIds size.");
- return this;
- }
-
- /**
- * Sets the Kafka bootstrap servers.
- *
- * @param bootstrapServers a comma separated list of valid URIs to reach the Kafka broker
- * @return {@link KafkaSinkBuilder}
- */
- public KafkaSinkBuilder<IN> setBootstrapServers(String bootstrapServers) {
- return setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
- }
-
- private void sanityCheck() {
- checkNotNull(
- kafkaProducerConfig.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG),
- "bootstrapServers");
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- checkState(
- transactionalIdPrefix != null,
- "EXACTLY_ONCE delivery guarantee requires a transactionIdPrefix to be set to provide unique transaction names across multiple KafkaSinks writing to the same Kafka cluster.");
- }
- checkNotNull(recordSerializer, "recordSerializer");
- }
-
- /**
- * Constructs the {@link KafkaSink} with the configured properties.
- *
- * @return {@link KafkaSink}
- */
- public KafkaSink<IN> build() {
- sanityCheck();
- return new KafkaSink<>(
- deliveryGuarantee, kafkaProducerConfig, transactionalIdPrefix, recordSerializer);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriter.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriter.java
deleted file mode 100644
index 29d7cb4984f..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriter.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.operators.MailboxExecutor;
-import org.apache.flink.api.common.operators.ProcessingTimeService;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.api.connector.sink2.StatefulSink;
-import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
-import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.connector.kafka.MetricUtil;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
-import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
-import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricMutableWrapper;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
-import org.apache.flink.shaded.guava30.com.google.common.io.Closer;
-
-import org.apache.kafka.clients.producer.Callback;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.errors.ProducerFencedException;
-import org.apache.kafka.common.errors.UnknownProducerIdException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.function.Consumer;
-
-import static org.apache.flink.util.IOUtils.closeAll;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * This class is responsible to write records in a Kafka topic and to handle the different delivery
- * {@link DeliveryGuarantee}s.
- *
- * @param <IN> The type of the input elements.
- */
-class KafkaWriter<IN>
- implements StatefulSink.StatefulSinkWriter<IN, KafkaWriterState>,
- TwoPhaseCommittingSink.PrecommittingSinkWriter<IN, KafkaCommittable> {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaWriter.class);
- private static final String KAFKA_PRODUCER_METRIC_NAME = "KafkaProducer";
- private static final long METRIC_UPDATE_INTERVAL_MILLIS = 500;
-
- private static final String KEY_DISABLE_METRICS = "flink.disable-metrics";
- private static final String KEY_REGISTER_METRICS = "register.producer.metrics";
- private static final String KAFKA_PRODUCER_METRICS = "producer-metrics";
-
- private final DeliveryGuarantee deliveryGuarantee;
- private final Properties kafkaProducerConfig;
- private final String transactionalIdPrefix;
- private final KafkaRecordSerializationSchema<IN> recordSerializer;
- private final Callback deliveryCallback;
- private final KafkaRecordSerializationSchema.KafkaSinkContext kafkaSinkContext;
- private volatile Exception asyncProducerException;
- private final Map<String, KafkaMetricMutableWrapper> previouslyCreatedMetrics = new HashMap<>();
- private final SinkWriterMetricGroup metricGroup;
- private final boolean disabledMetrics;
- private final Counter numRecordsOutCounter;
- private final Counter numBytesOutCounter;
- private final Counter numRecordsOutErrorsCounter;
- private final ProcessingTimeService timeService;
-
- // Number of outgoing bytes at the latest metric sync
- private long latestOutgoingByteTotal;
- private Metric byteOutMetric;
- private FlinkKafkaInternalProducer<byte[], byte[]> currentProducer;
- private final KafkaWriterState kafkaWriterState;
- // producer pool only used for exactly once
- private final Deque<FlinkKafkaInternalProducer<byte[], byte[]>> producerPool =
- new ArrayDeque<>();
- private final Closer closer = Closer.create();
- private long lastCheckpointId;
-
- private boolean closed = false;
- private long lastSync = System.currentTimeMillis();
-
- /**
- * Constructor creating a kafka writer.
- *
- * <p>It will throw a {@link RuntimeException} if {@link
- * KafkaRecordSerializationSchema#open(SerializationSchema.InitializationContext,
- * KafkaRecordSerializationSchema.KafkaSinkContext)} fails.
- *
- * @param deliveryGuarantee the Sink's delivery guarantee
- * @param kafkaProducerConfig the properties to configure the {@link FlinkKafkaInternalProducer}
- * @param transactionalIdPrefix used to create the transactionalIds
- * @param sinkInitContext context to provide information about the runtime environment
- * @param recordSerializer serialize to transform the incoming records to {@link ProducerRecord}
- * @param schemaContext context used to initialize the {@link KafkaRecordSerializationSchema}
- * @param recoveredStates state from an previous execution which was covered
- */
- KafkaWriter(
- DeliveryGuarantee deliveryGuarantee,
- Properties kafkaProducerConfig,
- String transactionalIdPrefix,
- Sink.InitContext sinkInitContext,
- KafkaRecordSerializationSchema<IN> recordSerializer,
- SerializationSchema.InitializationContext schemaContext,
- Collection<KafkaWriterState> recoveredStates) {
- this.deliveryGuarantee = checkNotNull(deliveryGuarantee, "deliveryGuarantee");
- this.kafkaProducerConfig = checkNotNull(kafkaProducerConfig, "kafkaProducerConfig");
- this.transactionalIdPrefix = checkNotNull(transactionalIdPrefix, "transactionalIdPrefix");
- this.recordSerializer = checkNotNull(recordSerializer, "recordSerializer");
- checkNotNull(sinkInitContext, "sinkInitContext");
- this.deliveryCallback =
- new WriterCallback(
- sinkInitContext.getMailboxExecutor(),
- sinkInitContext.<RecordMetadata>metadataConsumer().orElse(null));
- this.disabledMetrics =
- kafkaProducerConfig.containsKey(KEY_DISABLE_METRICS)
- && Boolean.parseBoolean(
- kafkaProducerConfig.get(KEY_DISABLE_METRICS).toString())
- || kafkaProducerConfig.containsKey(KEY_REGISTER_METRICS)
- && !Boolean.parseBoolean(
- kafkaProducerConfig.get(KEY_REGISTER_METRICS).toString());
- this.timeService = sinkInitContext.getProcessingTimeService();
- this.metricGroup = sinkInitContext.metricGroup();
- this.numBytesOutCounter = metricGroup.getIOMetricGroup().getNumBytesOutCounter();
- this.numRecordsOutCounter = metricGroup.getIOMetricGroup().getNumRecordsOutCounter();
- this.numRecordsOutErrorsCounter = metricGroup.getNumRecordsOutErrorsCounter();
- this.kafkaSinkContext =
- new DefaultKafkaSinkContext(
- sinkInitContext.getSubtaskId(),
- sinkInitContext.getNumberOfParallelSubtasks(),
- kafkaProducerConfig);
- try {
- recordSerializer.open(schemaContext, kafkaSinkContext);
- } catch (Exception e) {
- throw new FlinkRuntimeException("Cannot initialize schema.", e);
- }
-
- this.kafkaWriterState = new KafkaWriterState(transactionalIdPrefix);
- this.lastCheckpointId =
- sinkInitContext
- .getRestoredCheckpointId()
- .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- abortLingeringTransactions(
- checkNotNull(recoveredStates, "recoveredStates"), lastCheckpointId + 1);
- this.currentProducer = getTransactionalProducer(lastCheckpointId + 1);
- this.currentProducer.beginTransaction();
- } else if (deliveryGuarantee == DeliveryGuarantee.AT_LEAST_ONCE
- || deliveryGuarantee == DeliveryGuarantee.NONE) {
- this.currentProducer = new FlinkKafkaInternalProducer<>(this.kafkaProducerConfig, null);
- closer.register(this.currentProducer);
- initKafkaMetrics(this.currentProducer);
- } else {
- throw new UnsupportedOperationException(
- "Unsupported Kafka writer semantic " + this.deliveryGuarantee);
- }
-
- initFlinkMetrics();
- }
-
- @Override
- public void write(@Nullable IN element, Context context) throws IOException {
- checkAsyncException();
- final ProducerRecord<byte[], byte[]> record =
- recordSerializer.serialize(element, kafkaSinkContext, context.timestamp());
- if (record != null) {
- currentProducer.send(record, deliveryCallback);
- numRecordsOutCounter.inc();
- }
- }
-
- @Override
- public void flush(boolean endOfInput) throws IOException, InterruptedException {
- if (deliveryGuarantee != DeliveryGuarantee.NONE || endOfInput) {
- LOG.debug("final flush={}", endOfInput);
- currentProducer.flush();
- }
-
- checkAsyncException();
- }
-
- @Override
- public Collection<KafkaCommittable> prepareCommit() {
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- final List<KafkaCommittable> committables =
- Collections.singletonList(
- KafkaCommittable.of(currentProducer, producerPool::add));
- LOG.debug("Committing {} committables.", committables);
- return committables;
- }
- return Collections.emptyList();
- }
-
- @Override
- public List<KafkaWriterState> snapshotState(long checkpointId) throws IOException {
- if (deliveryGuarantee == DeliveryGuarantee.EXACTLY_ONCE) {
- currentProducer = getTransactionalProducer(checkpointId + 1);
- currentProducer.beginTransaction();
- }
- return ImmutableList.of(kafkaWriterState);
- }
-
- @Override
- public void close() throws Exception {
- closed = true;
- LOG.debug("Closing writer with {}", currentProducer);
- closeAll(
- this::abortCurrentProducer,
- closer,
- producerPool::clear,
- () -> {
- checkState(currentProducer.isClosed());
- currentProducer = null;
- });
-
- // Rethrow exception for the case in which close is called before writer() and flush().
- checkAsyncException();
- }
-
- private void abortCurrentProducer() {
- if (currentProducer.isInTransaction()) {
- try {
- currentProducer.abortTransaction();
- } catch (ProducerFencedException e) {
- LOG.debug(
- "Producer {} fenced while aborting", currentProducer.getTransactionalId());
- }
- }
- }
-
- @VisibleForTesting
- Deque<FlinkKafkaInternalProducer<byte[], byte[]>> getProducerPool() {
- return producerPool;
- }
-
- @VisibleForTesting
- FlinkKafkaInternalProducer<byte[], byte[]> getCurrentProducer() {
- return currentProducer;
- }
-
- void abortLingeringTransactions(
- Collection<KafkaWriterState> recoveredStates, long startCheckpointId) {
- List<String> prefixesToAbort = Lists.newArrayList(transactionalIdPrefix);
-
- final Optional<KafkaWriterState> lastStateOpt = recoveredStates.stream().findFirst();
- if (lastStateOpt.isPresent()) {
- KafkaWriterState lastState = lastStateOpt.get();
- if (!lastState.getTransactionalIdPrefix().equals(transactionalIdPrefix)) {
- prefixesToAbort.add(lastState.getTransactionalIdPrefix());
- LOG.warn(
- "Transactional id prefix from previous execution {} has changed to {}.",
- lastState.getTransactionalIdPrefix(),
- transactionalIdPrefix);
- }
- }
-
- try (TransactionAborter transactionAborter =
- new TransactionAborter(
- kafkaSinkContext.getParallelInstanceId(),
- kafkaSinkContext.getNumberOfParallelInstances(),
- this::getOrCreateTransactionalProducer,
- producerPool::add)) {
- transactionAborter.abortLingeringTransactions(prefixesToAbort, startCheckpointId);
- }
- }
-
- /**
- * For each checkpoint we create new {@link FlinkKafkaInternalProducer} so that new transactions
- * will not clash with transactions created during previous checkpoints ({@code
- * producer.initTransactions()} assures that we obtain new producerId and epoch counters).
- *
- * <p>Ensures that all transaction ids in between lastCheckpointId and checkpointId are
- * initialized.
- */
- private FlinkKafkaInternalProducer<byte[], byte[]> getTransactionalProducer(long checkpointId) {
- checkState(
- checkpointId > lastCheckpointId,
- "Expected %s > %s",
- checkpointId,
- lastCheckpointId);
- FlinkKafkaInternalProducer<byte[], byte[]> producer = null;
- // in case checkpoints have been aborted, Flink would create non-consecutive transaction ids
- // this loop ensures that all gaps are filled with initialized (empty) transactions
- for (long id = lastCheckpointId + 1; id <= checkpointId; id++) {
- String transactionalId =
- TransactionalIdFactory.buildTransactionalId(
- transactionalIdPrefix, kafkaSinkContext.getParallelInstanceId(), id);
- producer = getOrCreateTransactionalProducer(transactionalId);
- }
- this.lastCheckpointId = checkpointId;
- assert producer != null;
- LOG.info("Created new transactional producer {}", producer.getTransactionalId());
- return producer;
- }
-
- private FlinkKafkaInternalProducer<byte[], byte[]> getOrCreateTransactionalProducer(
- String transactionalId) {
- FlinkKafkaInternalProducer<byte[], byte[]> producer = producerPool.poll();
- if (producer == null) {
- producer = new FlinkKafkaInternalProducer<>(kafkaProducerConfig, transactionalId);
- closer.register(producer);
- producer.initTransactions();
- initKafkaMetrics(producer);
- } else {
- producer.initTransactionId(transactionalId);
- }
- return producer;
- }
-
- private void initFlinkMetrics() {
- metricGroup.setCurrentSendTimeGauge(this::computeSendTime);
- registerMetricSync();
- }
-
- private void initKafkaMetrics(FlinkKafkaInternalProducer<byte[], byte[]> producer) {
- byteOutMetric =
- MetricUtil.getKafkaMetric(
- producer.metrics(), KAFKA_PRODUCER_METRICS, "outgoing-byte-total");
- if (disabledMetrics) {
- return;
- }
- final MetricGroup kafkaMetricGroup = metricGroup.addGroup(KAFKA_PRODUCER_METRIC_NAME);
- producer.metrics().entrySet().forEach(initMetric(kafkaMetricGroup));
- }
-
- private Consumer<Map.Entry<MetricName, ? extends Metric>> initMetric(
- MetricGroup kafkaMetricGroup) {
- return (entry) -> {
- final String name = entry.getKey().name();
- final Metric metric = entry.getValue();
- if (previouslyCreatedMetrics.containsKey(name)) {
- final KafkaMetricMutableWrapper wrapper = previouslyCreatedMetrics.get(name);
- wrapper.setKafkaMetric(metric);
- } else {
- final KafkaMetricMutableWrapper wrapper = new KafkaMetricMutableWrapper(metric);
- previouslyCreatedMetrics.put(name, wrapper);
- kafkaMetricGroup.gauge(name, wrapper);
- }
- };
- }
-
- private long computeSendTime() {
- FlinkKafkaInternalProducer<byte[], byte[]> producer = this.currentProducer;
- if (producer == null) {
- return -1L;
- }
- final Metric sendTime =
- MetricUtil.getKafkaMetric(
- producer.metrics(), KAFKA_PRODUCER_METRICS, "request-latency-avg");
- final Metric queueTime =
- MetricUtil.getKafkaMetric(
- producer.metrics(), KAFKA_PRODUCER_METRICS, "record-queue-time-avg");
- return ((Number) sendTime.metricValue()).longValue()
- + ((Number) queueTime.metricValue()).longValue();
- }
-
- private void registerMetricSync() {
- timeService.registerTimer(
- lastSync + METRIC_UPDATE_INTERVAL_MILLIS,
- (time) -> {
- if (closed) {
- return;
- }
- long outgoingBytesUntilNow = ((Number) byteOutMetric.metricValue()).longValue();
- long outgoingBytesSinceLastUpdate =
- outgoingBytesUntilNow - latestOutgoingByteTotal;
- numBytesOutCounter.inc(outgoingBytesSinceLastUpdate);
- latestOutgoingByteTotal = outgoingBytesUntilNow;
- lastSync = time;
- registerMetricSync();
- });
- }
-
- /**
- * This method should only be invoked in the mailbox thread since the counter is not volatile.
- * Logic needs to be invoked by write AND flush since we support various semantics.
- */
- private void checkAsyncException() throws IOException {
- // reset this exception since we could close the writer later on
- Exception e = asyncProducerException;
- if (e != null) {
-
- asyncProducerException = null;
- numRecordsOutErrorsCounter.inc();
- throw new IOException(
- "One or more Kafka Producer send requests have encountered exception", e);
- }
- }
-
- private class WriterCallback implements Callback {
- private final MailboxExecutor mailboxExecutor;
- @Nullable private final Consumer<RecordMetadata> metadataConsumer;
-
- public WriterCallback(
- MailboxExecutor mailboxExecutor,
- @Nullable Consumer<RecordMetadata> metadataConsumer) {
- this.mailboxExecutor = mailboxExecutor;
- this.metadataConsumer = metadataConsumer;
- }
-
- @Override
- public void onCompletion(RecordMetadata metadata, Exception exception) {
- if (exception != null) {
- FlinkKafkaInternalProducer<byte[], byte[]> producer =
- KafkaWriter.this.currentProducer;
-
- // Propagate the first exception since amount of exceptions could be large. Need to
- // do this in Producer IO thread since flush() guarantees that the future will
- // complete. The same guarantee does not hold for tasks executed in separate
- // executor e.g. mailbox executor. flush() needs to have the exception immediately
- // available to fail the checkpoint.
- if (asyncProducerException == null) {
- asyncProducerException = decorateException(metadata, exception, producer);
- }
-
- mailboxExecutor.submit(
- () -> {
- // Checking for exceptions from previous writes
- checkAsyncException();
- },
- "Update error metric");
- }
-
- if (metadataConsumer != null) {
- metadataConsumer.accept(metadata);
- }
- }
-
- private FlinkRuntimeException decorateException(
- RecordMetadata metadata,
- Exception exception,
- FlinkKafkaInternalProducer<byte[], byte[]> producer) {
- String message =
- String.format("Failed to send data to Kafka %s with %s ", metadata, producer);
- if (exception instanceof UnknownProducerIdException) {
- message += KafkaCommitter.UNKNOWN_PRODUCER_ID_ERROR_MESSAGE;
- }
- return new FlinkRuntimeException(message, exception);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterState.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterState.java
deleted file mode 100644
index b4482c6915a..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterState.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import java.util.Objects;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-class KafkaWriterState {
- private final String transactionalIdPrefix;
-
- KafkaWriterState(String transactionalIdPrefix) {
- this.transactionalIdPrefix = checkNotNull(transactionalIdPrefix, "transactionalIdPrefix");
- }
-
- public String getTransactionalIdPrefix() {
- return transactionalIdPrefix;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- KafkaWriterState that = (KafkaWriterState) o;
- return transactionalIdPrefix.equals(that.transactionalIdPrefix);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(transactionalIdPrefix);
- }
-
- @Override
- public String toString() {
- return "KafkaWriterState{"
- + ", transactionalIdPrefix='"
- + transactionalIdPrefix
- + '\''
- + '}';
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterStateSerializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterStateSerializer.java
deleted file mode 100644
index 5c91967c5c8..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/KafkaWriterStateSerializer.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/** A serializer used to serialize {@link KafkaWriterState}. */
-class KafkaWriterStateSerializer implements SimpleVersionedSerializer<KafkaWriterState> {
-
- @Override
- public int getVersion() {
- return 1;
- }
-
- @Override
- public byte[] serialize(KafkaWriterState state) throws IOException {
- try (final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- final DataOutputStream out = new DataOutputStream(baos)) {
- out.writeUTF(state.getTransactionalIdPrefix());
- out.flush();
- return baos.toByteArray();
- }
- }
-
- @Override
- public KafkaWriterState deserialize(int version, byte[] serialized) throws IOException {
- try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- final DataInputStream in = new DataInputStream(bais)) {
- final String transactionalIdPrefx = in.readUTF();
- return new KafkaWriterState(transactionalIdPrefx);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/Recyclable.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/Recyclable.java
deleted file mode 100644
index 012fa992ca3..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/Recyclable.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import java.io.Closeable;
-import java.util.function.Consumer;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-class Recyclable<T> implements Closeable {
- private T object;
- private final Consumer<T> recycler;
-
- public Recyclable(T object, Consumer<T> recycler) {
- this.object = checkNotNull(object);
- this.recycler = checkNotNull(recycler);
- }
-
- public T getObject() {
- checkState(!isRecycled(), "Already recycled");
- return object;
- }
-
- boolean isRecycled() {
- return object == null;
- }
-
- @Override
- public void close() {
- recycler.accept(object);
- object = null;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TopicSelector.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TopicSelector.java
deleted file mode 100644
index 2a20754d901..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TopicSelector.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-import java.io.Serializable;
-import java.util.function.Function;
-
-/**
- * Selects a topic for the incoming record.
- *
- * @param <IN> type of the incoming record
- */
-@PublicEvolving
-public interface TopicSelector<IN> extends Function<IN, String>, Serializable {}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionAborter.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionAborter.java
deleted file mode 100644
index cae6ca6481a..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionAborter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import javax.annotation.Nullable;
-
-import java.io.Closeable;
-import java.util.List;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * Aborts lingering transactions on restart.
- *
- * <p>Transactions are lingering if they are not tracked anywhere. For example, if a job is started
- * transactions are opened. A restart without checkpoint would not allow Flink to abort old
- * transactions. Since Kafka's transactions are sequential, newly produced data does not become
- * visible for read_committed consumers. However, Kafka has no API for querying open transactions,
- * so they become lingering.
- *
- * <p>Flink solves this by assuming consecutive transaction ids. On restart of checkpoint C on
- * subtask S, it will sequentially cancel transaction C+1, C+2, ... of S until it finds the first
- * unused transaction.
- *
- * <p>Additionally, to cover for weird downscaling cases without checkpoints, it also checks for
- * transactions of subtask S+P where P is the current parallelism until it finds a subtask without
- * transactions.
- */
-class TransactionAborter implements Closeable {
- private final int subtaskId;
- private final int parallelism;
- private final Function<String, FlinkKafkaInternalProducer<byte[], byte[]>> producerFactory;
- private final Consumer<FlinkKafkaInternalProducer<byte[], byte[]>> closeAction;
- @Nullable FlinkKafkaInternalProducer<byte[], byte[]> producer = null;
-
- public TransactionAborter(
- int subtaskId,
- int parallelism,
- Function<String, FlinkKafkaInternalProducer<byte[], byte[]>> producerFactory,
- Consumer<FlinkKafkaInternalProducer<byte[], byte[]>> closeAction) {
- this.subtaskId = subtaskId;
- this.parallelism = parallelism;
- this.producerFactory = checkNotNull(producerFactory);
- this.closeAction = closeAction;
- }
-
- void abortLingeringTransactions(List<String> prefixesToAbort, long startCheckpointId) {
- for (String prefix : prefixesToAbort) {
- abortTransactionsWithPrefix(prefix, startCheckpointId);
- }
- }
-
- /**
- * Aborts all transactions that have been created by this subtask in a previous run.
- *
- * <p>It also aborts transactions from subtasks that may have been removed because of
- * downscaling.
- *
- * <p>When Flink downscales X subtasks to Y subtasks, then subtask i is responsible for cleaning
- * all subtasks j in [0; X), where j % Y = i. For example, if we downscale to 2, then subtask 0
- * is responsible for all even and subtask 1 for all odd subtasks.
- */
- private void abortTransactionsWithPrefix(String prefix, long startCheckpointId) {
- for (int subtaskId = this.subtaskId; ; subtaskId += parallelism) {
- if (abortTransactionOfSubtask(prefix, startCheckpointId, subtaskId) == 0) {
- // If Flink didn't abort any transaction for current subtask, then we assume that no
- // such subtask existed and no subtask with a higher number as well.
- break;
- }
- }
- }
-
- /**
- * Aborts all transactions that have been created by a subtask in a previous run after the given
- * checkpoint id.
- *
- * <p>We assume that transaction ids are consecutively used and thus Flink can stop aborting as
- * soon as Flink notices that a particular transaction id was unused.
- */
- private int abortTransactionOfSubtask(String prefix, long startCheckpointId, int subtaskId) {
- int numTransactionAborted = 0;
- for (long checkpointId = startCheckpointId; ; checkpointId++, numTransactionAborted++) {
- // initTransactions fences all old transactions with the same id by bumping the epoch
- String transactionalId =
- TransactionalIdFactory.buildTransactionalId(prefix, subtaskId, checkpointId);
- if (producer == null) {
- producer = producerFactory.apply(transactionalId);
- } else {
- producer.initTransactionId(transactionalId);
- }
- producer.flush();
- // An epoch of 0 indicates that the id was unused before
- if (producer.getEpoch() == 0) {
- // Note that the check works beyond transaction log timeouts and just depends on the
- // retention of the transaction topic (typically 7d). Any transaction that is not in
- // the that topic anymore is also not lingering (i.e., it will not block downstream
- // from reading)
- // This method will only cease to work if transaction log timeout = topic retention
- // and a user didn't restart the application for that period of time. Then the first
- // transactions would vanish from the topic while later transactions are still
- // lingering until they are cleaned up by Kafka. Then the user has to wait until the
- // other transactions are timed out (which shouldn't take too long).
- break;
- }
- }
- return numTransactionAborted;
- }
-
- public void close() {
- if (producer != null) {
- closeAction.accept(producer);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionalIdFactory.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionalIdFactory.java
deleted file mode 100644
index eda4c018098..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionalIdFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-class TransactionalIdFactory {
- private static final String TRANSACTIONAL_ID_DELIMITER = "-";
-
- /**
- * Constructs a transactionalId with the following format {@code
- * transactionalIdPrefix-subtaskId-checkpointOffset}.
- *
- * @param transactionalIdPrefix prefix for the id
- * @param subtaskId describing the subtask which is opening the transaction
- * @param checkpointOffset an always incrementing number usually capturing the number of
- * checkpoints taken by the subtask
- * @return transactionalId
- */
- public static String buildTransactionalId(
- String transactionalIdPrefix, int subtaskId, long checkpointOffset) {
- return transactionalIdPrefix
- + TRANSACTIONAL_ID_DELIMITER
- + subtaskId
- + TRANSACTIONAL_ID_DELIMITER
- + checkpointOffset;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionsToAbortChecker.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionsToAbortChecker.java
deleted file mode 100644
index 2d79b350654..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/sink/TransactionsToAbortChecker.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.sink;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-class TransactionsToAbortChecker {
-
- private static final int MINIMUM_CHECKPOINT_OFFSET = 1;
-
- private final int numberOfParallelSubtasks;
- private final Map<Integer, Long> subtaskIdCheckpointOffsetMapping;
- private final int subtaskId;
-
- TransactionsToAbortChecker(
- int numberOfParallelSubtasks,
- Map<Integer, Long> subtaskIdCheckpointOffsetMapping,
- int subtaskId) {
- this.subtaskId = subtaskId;
- this.numberOfParallelSubtasks = numberOfParallelSubtasks;
- this.subtaskIdCheckpointOffsetMapping = subtaskIdCheckpointOffsetMapping;
- }
-
- /**
- * Iterates through all open transactions and filters for the following attributes.
- *
- * <ol>
- * <li>If the minimum checkpointOffset for the subtask is {@link #MINIMUM_CHECKPOINT_OFFSET}
- * and [openSubtaskId % {@link #numberOfParallelSubtasks} == {@link #subtaskId}] return
- * all transactions from this subtask
- * <li>If the subtaskId is part of the recovered states {@link
- * #subtaskIdCheckpointOffsetMapping} and the checkpointOffset >= the recovered offSet
- * also return this transactionalId
- * </ol>
- *
- * @param openTransactions Mapping of {subtaskId: {checkpointOffset: transactionalId}}
- * @return transactionalIds which must be aborted
- */
- public List<String> getTransactionsToAbort(Map<Integer, Map<Long, String>> openTransactions) {
- final List<String> transactionalIdsToAbort = new ArrayList<>();
- for (final Map.Entry<Integer, Map<Long, String>> subtaskOffsetMapping :
- openTransactions.entrySet()) {
- final Map<Long, String> checkpointOffsetTransactionalIdMapping =
- subtaskOffsetMapping.getValue();
- // All transactions from this subtask have been closed
- if (checkpointOffsetTransactionalIdMapping.isEmpty()) {
- continue;
- }
- // Abort all open transactions if checkpointOffset 0 is open implying that no checkpoint
- // finished.
- // Cut the transactions in ranges to speed up abort process
- if (Collections.min(checkpointOffsetTransactionalIdMapping.keySet())
- == MINIMUM_CHECKPOINT_OFFSET
- && subtaskOffsetMapping.getKey() % numberOfParallelSubtasks == subtaskId) {
- transactionalIdsToAbort.addAll(checkpointOffsetTransactionalIdMapping.values());
- } else {
- // Check all open transactions against recovered ones and close if the open
- // transaction is equal or higher to the offset
- for (final Map.Entry<Long, String> offsetTransactionId :
- checkpointOffsetTransactionalIdMapping.entrySet()) {
- if (!hasSameSubtaskWithHigherCheckpoint(
- subtaskOffsetMapping.getKey(), offsetTransactionId.getKey())) {
- continue;
- }
- transactionalIdsToAbort.add(offsetTransactionId.getValue());
- }
- }
- }
- return transactionalIdsToAbort;
- }
-
- private boolean hasSameSubtaskWithHigherCheckpoint(
- int openSubtaskIndex, long openCheckpointOffset) {
- return subtaskIdCheckpointOffsetMapping.containsKey(openSubtaskIndex)
- && subtaskIdCheckpointOffsetMapping.get(openSubtaskIndex) <= openCheckpointOffset;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSource.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSource.java
deleted file mode 100644
index 7a17b1ff6f9..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSource.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.connector.source.Boundedness;
-import org.apache.flink.api.connector.source.Source;
-import org.apache.flink.api.connector.source.SourceReader;
-import org.apache.flink.api.connector.source.SourceReaderContext;
-import org.apache.flink.api.connector.source.SplitEnumerator;
-import org.apache.flink.api.connector.source.SplitEnumeratorContext;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
-import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumState;
-import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumStateSerializer;
-import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumerator;
-import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
-import org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriber;
-import org.apache.flink.connector.kafka.source.metrics.KafkaSourceReaderMetrics;
-import org.apache.flink.connector.kafka.source.reader.KafkaPartitionSplitReader;
-import org.apache.flink.connector.kafka.source.reader.KafkaRecordEmitter;
-import org.apache.flink.connector.kafka.source.reader.KafkaSourceReader;
-import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema;
-import org.apache.flink.connector.kafka.source.reader.fetcher.KafkaSourceFetcherManager;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitSerializer;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.util.UserCodeClassLoader;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Properties;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-/**
- * The Source implementation of Kafka. Please use a {@link KafkaSourceBuilder} to construct a {@link
- * KafkaSource}. The following example shows how to create a KafkaSource emitting records of <code>
- * String</code> type.
- *
- * <pre>{@code
- * KafkaSource<String> source = KafkaSource
- * .<String>builder()
- * .setBootstrapServers(KafkaSourceTestEnv.brokerConnectionStrings)
- * .setGroupId("MyGroup")
- * .setTopics(Arrays.asList(TOPIC1, TOPIC2))
- * .setDeserializer(new TestingKafkaRecordDeserializationSchema())
- * .setStartingOffsets(OffsetsInitializer.earliest())
- * .build();
- * }</pre>
- *
- * <p>{@link org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumerator} only supports
- * adding new splits and not removing splits in split discovery.
- *
- * <p>See {@link KafkaSourceBuilder} for more details on how to configure this source.
- *
- * @param <OUT> the output type of the source.
- */
-@PublicEvolving
-public class KafkaSource<OUT>
- implements Source<OUT, KafkaPartitionSplit, KafkaSourceEnumState>,
- ResultTypeQueryable<OUT> {
- private static final long serialVersionUID = -8755372893283732098L;
- // Users can choose only one of the following ways to specify the topics to consume from.
- private final KafkaSubscriber subscriber;
- // Users can specify the starting / stopping offset initializer.
- private final OffsetsInitializer startingOffsetsInitializer;
- private final OffsetsInitializer stoppingOffsetsInitializer;
- // Boundedness
- private final Boundedness boundedness;
- private final KafkaRecordDeserializationSchema<OUT> deserializationSchema;
- // The configurations.
- private final Properties props;
-
- KafkaSource(
- KafkaSubscriber subscriber,
- OffsetsInitializer startingOffsetsInitializer,
- @Nullable OffsetsInitializer stoppingOffsetsInitializer,
- Boundedness boundedness,
- KafkaRecordDeserializationSchema<OUT> deserializationSchema,
- Properties props) {
- this.subscriber = subscriber;
- this.startingOffsetsInitializer = startingOffsetsInitializer;
- this.stoppingOffsetsInitializer = stoppingOffsetsInitializer;
- this.boundedness = boundedness;
- this.deserializationSchema = deserializationSchema;
- this.props = props;
- }
-
- /**
- * Get a kafkaSourceBuilder to build a {@link KafkaSource}.
- *
- * @return a Kafka source builder.
- */
- public static <OUT> KafkaSourceBuilder<OUT> builder() {
- return new KafkaSourceBuilder<>();
- }
-
- @Override
- public Boundedness getBoundedness() {
- return this.boundedness;
- }
-
- @Internal
- @Override
- public SourceReader<OUT, KafkaPartitionSplit> createReader(SourceReaderContext readerContext)
- throws Exception {
- return createReader(readerContext, (ignore) -> {});
- }
-
- @VisibleForTesting
- SourceReader<OUT, KafkaPartitionSplit> createReader(
- SourceReaderContext readerContext, Consumer<Collection<String>> splitFinishedHook)
- throws Exception {
- FutureCompletingBlockingQueue<RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>>>
- elementsQueue = new FutureCompletingBlockingQueue<>();
- deserializationSchema.open(
- new DeserializationSchema.InitializationContext() {
- @Override
- public MetricGroup getMetricGroup() {
- return readerContext.metricGroup().addGroup("deserializer");
- }
-
- @Override
- public UserCodeClassLoader getUserCodeClassLoader() {
- return readerContext.getUserCodeClassLoader();
- }
- });
- final KafkaSourceReaderMetrics kafkaSourceReaderMetrics =
- new KafkaSourceReaderMetrics(readerContext.metricGroup());
-
- Supplier<KafkaPartitionSplitReader> splitReaderSupplier =
- () -> new KafkaPartitionSplitReader(props, readerContext, kafkaSourceReaderMetrics);
- KafkaRecordEmitter<OUT> recordEmitter = new KafkaRecordEmitter<>(deserializationSchema);
-
- return new KafkaSourceReader<>(
- elementsQueue,
- new KafkaSourceFetcherManager(
- elementsQueue, splitReaderSupplier::get, splitFinishedHook),
- recordEmitter,
- toConfiguration(props),
- readerContext,
- kafkaSourceReaderMetrics);
- }
-
- @Internal
- @Override
- public SplitEnumerator<KafkaPartitionSplit, KafkaSourceEnumState> createEnumerator(
- SplitEnumeratorContext<KafkaPartitionSplit> enumContext) {
- return new KafkaSourceEnumerator(
- subscriber,
- startingOffsetsInitializer,
- stoppingOffsetsInitializer,
- props,
- enumContext,
- boundedness);
- }
-
- @Internal
- @Override
- public SplitEnumerator<KafkaPartitionSplit, KafkaSourceEnumState> restoreEnumerator(
- SplitEnumeratorContext<KafkaPartitionSplit> enumContext,
- KafkaSourceEnumState checkpoint)
- throws IOException {
- return new KafkaSourceEnumerator(
- subscriber,
- startingOffsetsInitializer,
- stoppingOffsetsInitializer,
- props,
- enumContext,
- boundedness,
- checkpoint.assignedPartitions());
- }
-
- @Internal
- @Override
- public SimpleVersionedSerializer<KafkaPartitionSplit> getSplitSerializer() {
- return new KafkaPartitionSplitSerializer();
- }
-
- @Internal
- @Override
- public SimpleVersionedSerializer<KafkaSourceEnumState> getEnumeratorCheckpointSerializer() {
- return new KafkaSourceEnumStateSerializer();
- }
-
- @Override
- public TypeInformation<OUT> getProducedType() {
- return deserializationSchema.getProducedType();
- }
-
- // ----------- private helper methods ---------------
-
- private Configuration toConfiguration(Properties props) {
- Configuration config = new Configuration();
- props.stringPropertyNames().forEach(key -> config.setString(key, props.getProperty(key)));
- return config;
- }
-
- @VisibleForTesting
- Configuration getConfiguration() {
- return toConfiguration(props);
- }
-
- @VisibleForTesting
- KafkaSubscriber getKafkaSubscriber() {
- return subscriber;
- }
-
- @VisibleForTesting
- OffsetsInitializer getStoppingOffsetsInitializer() {
- return stoppingOffsetsInitializer;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceBuilder.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceBuilder.java
deleted file mode 100644
index afaa72db71f..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceBuilder.java
+++ /dev/null
@@ -1,534 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.connector.source.Boundedness;
-import org.apache.flink.connector.kafka.source.enumerator.initializer.NoStoppingOffsetsInitializer;
-import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
-import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializerValidator;
-import org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriber;
-import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * The builder class for {@link KafkaSource} to make it easier for the users to construct a {@link
- * KafkaSource}.
- *
- * <p>The following example shows the minimum setup to create a KafkaSource that reads the String
- * values from a Kafka topic.
- *
- * <pre>{@code
- * KafkaSource<String> source = KafkaSource
- * .<String>builder()
- * .setBootstrapServers(MY_BOOTSTRAP_SERVERS)
- * .setTopics(Arrays.asList(TOPIC1, TOPIC2))
- * .setDeserializer(KafkaRecordDeserializationSchema.valueOnly(StringDeserializer.class))
- * .build();
- * }</pre>
- *
- * <p>The bootstrap servers, topics/partitions to consume, and the record deserializer are required
- * fields that must be set.
- *
- * <p>To specify the starting offsets of the KafkaSource, one can call {@link
- * #setStartingOffsets(OffsetsInitializer)}.
- *
- * <p>By default the KafkaSource runs in an {@link Boundedness#CONTINUOUS_UNBOUNDED} mode and never
- * stops until the Flink job is canceled or fails. To let the KafkaSource run as {@link
- * Boundedness#CONTINUOUS_UNBOUNDED} yet stop at some given offsets, one can call {@link
- * #setUnbounded(OffsetsInitializer)}. For example the following KafkaSource stops after it consumes
- * up to the latest partition offsets at the point when the Flink job started.
- *
- * <pre>{@code
- * KafkaSource<String> source = KafkaSource
- * .<String>builder()
- * .setBootstrapServers(MY_BOOTSTRAP_SERVERS)
- * .setTopics(Arrays.asList(TOPIC1, TOPIC2))
- * .setDeserializer(KafkaRecordDeserializationSchema.valueOnly(StringDeserializer.class))
- * .setUnbounded(OffsetsInitializer.latest())
- * .build();
- * }</pre>
- *
- * <p>Check the Java docs of each individual methods to learn more about the settings to build a
- * KafkaSource.
- */
-@PublicEvolving
-public class KafkaSourceBuilder<OUT> {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceBuilder.class);
- private static final String[] REQUIRED_CONFIGS = {ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG};
- // The subscriber specifies the partitions to subscribe to.
- private KafkaSubscriber subscriber;
- // Users can specify the starting / stopping offset initializer.
- private OffsetsInitializer startingOffsetsInitializer;
- private OffsetsInitializer stoppingOffsetsInitializer;
- // Boundedness
- private Boundedness boundedness;
- private KafkaRecordDeserializationSchema<OUT> deserializationSchema;
- // The configurations.
- protected Properties props;
-
- KafkaSourceBuilder() {
- this.subscriber = null;
- this.startingOffsetsInitializer = OffsetsInitializer.earliest();
- this.stoppingOffsetsInitializer = new NoStoppingOffsetsInitializer();
- this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
- this.deserializationSchema = null;
- this.props = new Properties();
- }
-
- /**
- * Sets the bootstrap servers for the KafkaConsumer of the KafkaSource.
- *
- * @param bootstrapServers the bootstrap servers of the Kafka cluster.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setBootstrapServers(String bootstrapServers) {
- return setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
- }
-
- /**
- * Sets the consumer group id of the KafkaSource.
- *
- * @param groupId the group id of the KafkaSource.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setGroupId(String groupId) {
- return setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
- }
-
- /**
- * Set a list of topics the KafkaSource should consume from. All the topics in the list should
- * have existed in the Kafka cluster. Otherwise an exception will be thrown. To allow some of
- * the topics to be created lazily, please use {@link #setTopicPattern(Pattern)} instead.
- *
- * @param topics the list of topics to consume from.
- * @return this KafkaSourceBuilder.
- * @see org.apache.kafka.clients.consumer.KafkaConsumer#subscribe(Collection)
- */
- public KafkaSourceBuilder<OUT> setTopics(List<String> topics) {
- ensureSubscriberIsNull("topics");
- subscriber = KafkaSubscriber.getTopicListSubscriber(topics);
- return this;
- }
-
- /**
- * Set a list of topics the KafkaSource should consume from. All the topics in the list should
- * have existed in the Kafka cluster. Otherwise an exception will be thrown. To allow some of
- * the topics to be created lazily, please use {@link #setTopicPattern(Pattern)} instead.
- *
- * @param topics the list of topics to consume from.
- * @return this KafkaSourceBuilder.
- * @see org.apache.kafka.clients.consumer.KafkaConsumer#subscribe(Collection)
- */
- public KafkaSourceBuilder<OUT> setTopics(String... topics) {
- return setTopics(Arrays.asList(topics));
- }
-
- /**
- * Set a topic pattern to consume from use the java {@link Pattern}.
- *
- * @param topicPattern the pattern of the topic name to consume from.
- * @return this KafkaSourceBuilder.
- * @see org.apache.kafka.clients.consumer.KafkaConsumer#subscribe(Pattern)
- */
- public KafkaSourceBuilder<OUT> setTopicPattern(Pattern topicPattern) {
- ensureSubscriberIsNull("topic pattern");
- subscriber = KafkaSubscriber.getTopicPatternSubscriber(topicPattern);
- return this;
- }
-
- /**
- * Set a set of partitions to consume from.
- *
- * @param partitions the set of partitions to consume from.
- * @return this KafkaSourceBuilder.
- * @see org.apache.kafka.clients.consumer.KafkaConsumer#assign(Collection)
- */
- public KafkaSourceBuilder<OUT> setPartitions(Set<TopicPartition> partitions) {
- ensureSubscriberIsNull("partitions");
- subscriber = KafkaSubscriber.getPartitionSetSubscriber(partitions);
- return this;
- }
-
- /**
- * Set a custom Kafka subscriber to use to discover new splits.
- *
- * @param kafkaSubscriber the {@link KafkaSubscriber} to use for split discovery.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setKafkaSubscriber(KafkaSubscriber kafkaSubscriber) {
- ensureSubscriberIsNull("custom");
- this.subscriber = checkNotNull(kafkaSubscriber);
- return this;
- }
-
- /**
- * Specify from which offsets the KafkaSource should start consuming from by providing an {@link
- * OffsetsInitializer}.
- *
- * <p>The following {@link OffsetsInitializer}s are commonly used and provided out of the box.
- * Users can also implement their own {@link OffsetsInitializer} for custom behaviors.
- *
- * <ul>
- * <li>{@link OffsetsInitializer#earliest()} - starting from the earliest offsets. This is
- * also the default {@link OffsetsInitializer} of the KafkaSource for starting offsets.
- * <li>{@link OffsetsInitializer#latest()} - starting from the latest offsets.
- * <li>{@link OffsetsInitializer#committedOffsets()} - starting from the committed offsets of
- * the consumer group.
- * <li>{@link
- * OffsetsInitializer#committedOffsets(org.apache.kafka.clients.consumer.OffsetResetStrategy)}
- * - starting from the committed offsets of the consumer group. If there is no committed
- * offsets, starting from the offsets specified by the {@link
- * org.apache.kafka.clients.consumer.OffsetResetStrategy OffsetResetStrategy}.
- * <li>{@link OffsetsInitializer#offsets(Map)} - starting from the specified offsets for each
- * partition.
- * <li>{@link OffsetsInitializer#timestamp(long)} - starting from the specified timestamp for
- * each partition. Note that the guarantee here is that all the records in Kafka whose
- * {@link org.apache.kafka.clients.consumer.ConsumerRecord#timestamp()} is greater than
- * the given starting timestamp will be consumed. However, it is possible that some
- * consumer records whose timestamp is smaller than the given starting timestamp are also
- * consumed.
- * </ul>
- *
- * @param startingOffsetsInitializer the {@link OffsetsInitializer} setting the starting offsets
- * for the Source.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setStartingOffsets(
- OffsetsInitializer startingOffsetsInitializer) {
- this.startingOffsetsInitializer = startingOffsetsInitializer;
- return this;
- }
-
- /**
- * By default the KafkaSource is set to run as {@link Boundedness#CONTINUOUS_UNBOUNDED} and thus
- * never stops until the Flink job fails or is canceled. To let the KafkaSource run as a
- * streaming source but still stop at some point, one can set an {@link OffsetsInitializer} to
- * specify the stopping offsets for each partition. When all the partitions have reached their
- * stopping offsets, the KafkaSource will then exit.
- *
- * <p>This method is different from {@link #setBounded(OffsetsInitializer)} in that after
- * setting the stopping offsets with this method, {@link KafkaSource#getBoundedness()} will
- * still return {@link Boundedness#CONTINUOUS_UNBOUNDED} even though it will stop at the
- * stopping offsets specified by the stopping offsets {@link OffsetsInitializer}.
- *
- * <p>The following {@link OffsetsInitializer} are commonly used and provided out of the box.
- * Users can also implement their own {@link OffsetsInitializer} for custom behaviors.
- *
- * <ul>
- * <li>{@link OffsetsInitializer#latest()} - stop at the latest offsets of the partitions when
- * the KafkaSource starts to run.
- * <li>{@link OffsetsInitializer#committedOffsets()} - stops at the committed offsets of the
- * consumer group.
- * <li>{@link OffsetsInitializer#offsets(Map)} - stops at the specified offsets for each
- * partition.
- * <li>{@link OffsetsInitializer#timestamp(long)} - stops at the specified timestamp for each
- * partition. The guarantee of setting the stopping timestamp is that no Kafka records
- * whose {@link org.apache.kafka.clients.consumer.ConsumerRecord#timestamp()} is greater
- * than the given stopping timestamp will be consumed. However, it is possible that some
- * records whose timestamp is smaller than the specified stopping timestamp are not
- * consumed.
- * </ul>
- *
- * @param stoppingOffsetsInitializer The {@link OffsetsInitializer} to specify the stopping
- * offset.
- * @return this KafkaSourceBuilder.
- * @see #setBounded(OffsetsInitializer)
- */
- public KafkaSourceBuilder<OUT> setUnbounded(OffsetsInitializer stoppingOffsetsInitializer) {
- this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
- this.stoppingOffsetsInitializer = stoppingOffsetsInitializer;
- return this;
- }
-
- /**
- * By default the KafkaSource is set to run as {@link Boundedness#CONTINUOUS_UNBOUNDED} and thus
- * never stops until the Flink job fails or is canceled. To let the KafkaSource run as {@link
- * Boundedness#BOUNDED} and stop at some point, one can set an {@link OffsetsInitializer} to
- * specify the stopping offsets for each partition. When all the partitions have reached their
- * stopping offsets, the KafkaSource will then exit.
- *
- * <p>This method is different from {@link #setUnbounded(OffsetsInitializer)} in that after
- * setting the stopping offsets with this method, {@link KafkaSource#getBoundedness()} will
- * return {@link Boundedness#BOUNDED} instead of {@link Boundedness#CONTINUOUS_UNBOUNDED}.
- *
- * <p>The following {@link OffsetsInitializer} are commonly used and provided out of the box.
- * Users can also implement their own {@link OffsetsInitializer} for custom behaviors.
- *
- * <ul>
- * <li>{@link OffsetsInitializer#latest()} - stop at the latest offsets of the partitions when
- * the KafkaSource starts to run.
- * <li>{@link OffsetsInitializer#committedOffsets()} - stops at the committed offsets of the
- * consumer group.
- * <li>{@link OffsetsInitializer#offsets(Map)} - stops at the specified offsets for each
- * partition.
- * <li>{@link OffsetsInitializer#timestamp(long)} - stops at the specified timestamp for each
- * partition. The guarantee of setting the stopping timestamp is that no Kafka records
- * whose {@link org.apache.kafka.clients.consumer.ConsumerRecord#timestamp()} is greater
- * than the given stopping timestamp will be consumed. However, it is possible that some
- * records whose timestamp is smaller than the specified stopping timestamp are not
- * consumed.
- * </ul>
- *
- * @param stoppingOffsetsInitializer the {@link OffsetsInitializer} to specify the stopping
- * offsets.
- * @return this KafkaSourceBuilder.
- * @see #setUnbounded(OffsetsInitializer)
- */
- public KafkaSourceBuilder<OUT> setBounded(OffsetsInitializer stoppingOffsetsInitializer) {
- this.boundedness = Boundedness.BOUNDED;
- this.stoppingOffsetsInitializer = stoppingOffsetsInitializer;
- return this;
- }
-
- /**
- * Sets the {@link KafkaRecordDeserializationSchema deserializer} of the {@link
- * org.apache.kafka.clients.consumer.ConsumerRecord ConsumerRecord} for KafkaSource.
- *
- * @param recordDeserializer the deserializer for Kafka {@link
- * org.apache.kafka.clients.consumer.ConsumerRecord ConsumerRecord}.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setDeserializer(
- KafkaRecordDeserializationSchema<OUT> recordDeserializer) {
- this.deserializationSchema = recordDeserializer;
- return this;
- }
-
- /**
- * Sets the {@link KafkaRecordDeserializationSchema deserializer} of the {@link
- * org.apache.kafka.clients.consumer.ConsumerRecord ConsumerRecord} for KafkaSource. The given
- * {@link DeserializationSchema} will be used to deserialize the value of ConsumerRecord. The
- * other information (e.g. key) in a ConsumerRecord will be ignored.
- *
- * @param deserializationSchema the {@link DeserializationSchema} to use for deserialization.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setValueOnlyDeserializer(
- DeserializationSchema<OUT> deserializationSchema) {
- this.deserializationSchema =
- KafkaRecordDeserializationSchema.valueOnly(deserializationSchema);
- return this;
- }
-
- /**
- * Sets the client id prefix of this KafkaSource.
- *
- * @param prefix the client id prefix to use for this KafkaSource.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setClientIdPrefix(String prefix) {
- return setProperty(KafkaSourceOptions.CLIENT_ID_PREFIX.key(), prefix);
- }
-
- /**
- * Set an arbitrary property for the KafkaSource and KafkaConsumer. The valid keys can be found
- * in {@link ConsumerConfig} and {@link KafkaSourceOptions}.
- *
- * <p>Note that the following keys will be overridden by the builder when the KafkaSource is
- * created.
- *
- * <ul>
- * <li><code>key.deserializer</code> is always set to {@link ByteArrayDeserializer}.
- * <li><code>value.deserializer</code> is always set to {@link ByteArrayDeserializer}.
- * <li><code>auto.offset.reset.strategy</code> is overridden by {@link
- * OffsetsInitializer#getAutoOffsetResetStrategy()} for the starting offsets, which is by
- * default {@link OffsetsInitializer#earliest()}.
- * <li><code>partition.discovery.interval.ms</code> is overridden to -1 when {@link
- * #setBounded(OffsetsInitializer)} has been invoked.
- * </ul>
- *
- * @param key the key of the property.
- * @param value the value of the property.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setProperty(String key, String value) {
- props.setProperty(key, value);
- return this;
- }
-
- /**
- * Set arbitrary properties for the KafkaSource and KafkaConsumer. The valid keys can be found
- * in {@link ConsumerConfig} and {@link KafkaSourceOptions}.
- *
- * <p>Note that the following keys will be overridden by the builder when the KafkaSource is
- * created.
- *
- * <ul>
- * <li><code>key.deserializer</code> is always set to {@link ByteArrayDeserializer}.
- * <li><code>value.deserializer</code> is always set to {@link ByteArrayDeserializer}.
- * <li><code>auto.offset.reset.strategy</code> is overridden by {@link
- * OffsetsInitializer#getAutoOffsetResetStrategy()} for the starting offsets, which is by
- * default {@link OffsetsInitializer#earliest()}.
- * <li><code>partition.discovery.interval.ms</code> is overridden to -1 when {@link
- * #setBounded(OffsetsInitializer)} has been invoked.
- * <li><code>client.id</code> is overridden to the "client.id.prefix-RANDOM_LONG", or
- * "group.id-RANDOM_LONG" if the client id prefix is not set.
- * </ul>
- *
- * @param props the properties to set for the KafkaSource.
- * @return this KafkaSourceBuilder.
- */
- public KafkaSourceBuilder<OUT> setProperties(Properties props) {
- this.props.putAll(props);
- return this;
- }
-
- /**
- * Build the {@link KafkaSource}.
- *
- * @return a KafkaSource with the settings made for this builder.
- */
- public KafkaSource<OUT> build() {
- sanityCheck();
- parseAndSetRequiredProperties();
- return new KafkaSource<>(
- subscriber,
- startingOffsetsInitializer,
- stoppingOffsetsInitializer,
- boundedness,
- deserializationSchema,
- props);
- }
-
- // ------------- private helpers --------------
-
- private void ensureSubscriberIsNull(String attemptingSubscribeMode) {
- if (subscriber != null) {
- throw new IllegalStateException(
- String.format(
- "Cannot use %s for consumption because a %s is already set for consumption.",
- attemptingSubscribeMode, subscriber.getClass().getSimpleName()));
- }
- }
-
- private void parseAndSetRequiredProperties() {
- maybeOverride(
- ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
- ByteArrayDeserializer.class.getName(),
- true);
- maybeOverride(
- ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
- ByteArrayDeserializer.class.getName(),
- true);
- if (!props.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
- LOG.warn(
- "Offset commit on checkpoint is disabled because {} is not specified",
- ConsumerConfig.GROUP_ID_CONFIG);
- maybeOverride(KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT.key(), "false", false);
- }
- maybeOverride(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false", false);
- maybeOverride(
- ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,
- startingOffsetsInitializer.getAutoOffsetResetStrategy().name().toLowerCase(),
- true);
-
- // If the source is bounded, do not run periodic partition discovery.
- maybeOverride(
- KafkaSourceOptions.PARTITION_DISCOVERY_INTERVAL_MS.key(),
- "-1",
- boundedness == Boundedness.BOUNDED);
-
- // If the client id prefix is not set, reuse the consumer group id as the client id prefix,
- // or generate a random string if consumer group id is not specified.
- maybeOverride(
- KafkaSourceOptions.CLIENT_ID_PREFIX.key(),
- props.containsKey(ConsumerConfig.GROUP_ID_CONFIG)
- ? props.getProperty(ConsumerConfig.GROUP_ID_CONFIG)
- : "KafkaSource-" + new Random().nextLong(),
- false);
- }
-
- private boolean maybeOverride(String key, String value, boolean override) {
- boolean overridden = false;
- String userValue = props.getProperty(key);
- if (userValue != null) {
- if (override) {
- LOG.warn(
- String.format(
- "Property %s is provided but will be overridden from %s to %s",
- key, userValue, value));
- props.setProperty(key, value);
- overridden = true;
- }
- } else {
- props.setProperty(key, value);
- }
- return overridden;
- }
-
- private void sanityCheck() {
- // Check required configs.
- for (String requiredConfig : REQUIRED_CONFIGS) {
- checkNotNull(
- props.getProperty(requiredConfig),
- String.format("Property %s is required but not provided", requiredConfig));
- }
- // Check required settings.
- checkNotNull(
- subscriber,
- "No subscribe mode is specified, "
- + "should be one of topics, topic pattern and partition set.");
- checkNotNull(deserializationSchema, "Deserialization schema is required but not provided.");
- // Check consumer group ID
- checkState(
- props.containsKey(ConsumerConfig.GROUP_ID_CONFIG) || !offsetCommitEnabledManually(),
- String.format(
- "Property %s is required when offset commit is enabled",
- ConsumerConfig.GROUP_ID_CONFIG));
- // Check offsets initializers
- if (startingOffsetsInitializer instanceof OffsetsInitializerValidator) {
- ((OffsetsInitializerValidator) startingOffsetsInitializer).validate(props);
- }
- if (stoppingOffsetsInitializer instanceof OffsetsInitializerValidator) {
- ((OffsetsInitializerValidator) stoppingOffsetsInitializer).validate(props);
- }
- }
-
- private boolean offsetCommitEnabledManually() {
- boolean autoCommit =
- props.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)
- && Boolean.parseBoolean(
- props.getProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
- boolean commitOnCheckpoint =
- props.containsKey(KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT.key())
- && Boolean.parseBoolean(
- props.getProperty(
- KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT.key()));
- return autoCommit || commitOnCheckpoint;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceOptions.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceOptions.java
deleted file mode 100644
index 248014c8a71..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/KafkaSourceOptions.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.configuration.ConfigOption;
-import org.apache.flink.configuration.ConfigOptions;
-
-import java.util.Properties;
-import java.util.function.Function;
-
-/** Configurations for KafkaSource. */
-@Internal
-public class KafkaSourceOptions {
-
- public static final ConfigOption<String> CLIENT_ID_PREFIX =
- ConfigOptions.key("client.id.prefix")
- .stringType()
- .noDefaultValue()
- .withDescription("The prefix to use for the Kafka consumers.");
-
- public static final ConfigOption<Long> PARTITION_DISCOVERY_INTERVAL_MS =
- ConfigOptions.key("partition.discovery.interval.ms")
- .longType()
- .noDefaultValue()
- .withDescription(
- "The interval in milliseconds for the Kafka source to discover "
- + "the new partitions. A non-positive value disables the partition discovery.");
-
- public static final ConfigOption<Boolean> REGISTER_KAFKA_CONSUMER_METRICS =
- ConfigOptions.key("register.consumer.metrics")
- .booleanType()
- .defaultValue(true)
- .withDescription(
- "Whether to register metrics of KafkaConsumer into Flink metric group");
-
- public static final ConfigOption<Boolean> COMMIT_OFFSETS_ON_CHECKPOINT =
- ConfigOptions.key("commit.offsets.on.checkpoint")
- .booleanType()
- .defaultValue(true)
- .withDescription("Whether to commit consuming offset on checkpoint.");
-
- @SuppressWarnings("unchecked")
- public static <T> T getOption(
- Properties props, ConfigOption<?> configOption, Function<String, T> parser) {
- String value = props.getProperty(configOption.key());
- return (T) (value == null ? configOption.defaultValue() : parser.apply(value));
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumState.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumState.java
deleted file mode 100644
index 04604abe14f..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumState.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator;
-
-import org.apache.flink.annotation.Internal;
-
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.Set;
-
-/** The state of Kafka source enumerator. */
-@Internal
-public class KafkaSourceEnumState {
- private final Set<TopicPartition> assignedPartitions;
-
- KafkaSourceEnumState(Set<TopicPartition> assignedPartitions) {
- this.assignedPartitions = assignedPartitions;
- }
-
- public Set<TopicPartition> assignedPartitions() {
- return assignedPartitions;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumStateSerializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumStateSerializer.java
deleted file mode 100644
index 7428f420509..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumStateSerializer.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.connector.base.source.utils.SerdeUtils;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitSerializer;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import org.apache.kafka.common.TopicPartition;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * The {@link org.apache.flink.core.io.SimpleVersionedSerializer Serializer} for the enumerator
- * state of Kafka source.
- */
-@Internal
-public class KafkaSourceEnumStateSerializer
- implements SimpleVersionedSerializer<KafkaSourceEnumState> {
-
- private static final int VERSION_0 = 0;
- private static final int VERSION_1 = 1;
-
- private static final int CURRENT_VERSION = VERSION_1;
-
- @Override
- public int getVersion() {
- return CURRENT_VERSION;
- }
-
- @Override
- public byte[] serialize(KafkaSourceEnumState enumState) throws IOException {
- return serializeTopicPartitions(enumState.assignedPartitions());
- }
-
- @Override
- public KafkaSourceEnumState deserialize(int version, byte[] serialized) throws IOException {
- if (version == CURRENT_VERSION) {
- final Set<TopicPartition> assignedPartitions = deserializeTopicPartitions(serialized);
- return new KafkaSourceEnumState(assignedPartitions);
- }
-
- // Backward compatibility
- if (version == VERSION_0) {
- Map<Integer, Set<KafkaPartitionSplit>> currentPartitionAssignment =
- SerdeUtils.deserializeSplitAssignments(
- serialized, new KafkaPartitionSplitSerializer(), HashSet::new);
- Set<TopicPartition> currentAssignedSplits = new HashSet<>();
- currentPartitionAssignment.forEach(
- (reader, splits) ->
- splits.forEach(
- split -> currentAssignedSplits.add(split.getTopicPartition())));
- return new KafkaSourceEnumState(currentAssignedSplits);
- }
-
- throw new IOException(
- String.format(
- "The bytes are serialized with version %d, "
- + "while this deserializer only supports version up to %d",
- version, CURRENT_VERSION));
- }
-
- private static byte[] serializeTopicPartitions(Collection<TopicPartition> topicPartitions)
- throws IOException {
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos)) {
-
- out.writeInt(topicPartitions.size());
- for (TopicPartition tp : topicPartitions) {
- out.writeUTF(tp.topic());
- out.writeInt(tp.partition());
- }
- out.flush();
-
- return baos.toByteArray();
- }
- }
-
- private static Set<TopicPartition> deserializeTopicPartitions(byte[] serializedTopicPartitions)
- throws IOException {
- try (ByteArrayInputStream bais = new ByteArrayInputStream(serializedTopicPartitions);
- DataInputStream in = new DataInputStream(bais)) {
-
- final int numPartitions = in.readInt();
- Set<TopicPartition> topicPartitions = new HashSet<>(numPartitions);
- for (int i = 0; i < numPartitions; i++) {
- final String topic = in.readUTF();
- final int partition = in.readInt();
- topicPartitions.add(new TopicPartition(topic, partition));
- }
- if (in.available() > 0) {
- throw new IOException("Unexpected trailing bytes in serialized topic partitions");
- }
-
- return topicPartitions;
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumerator.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumerator.java
deleted file mode 100644
index 9cf233c5029..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/KafkaSourceEnumerator.java
+++ /dev/null
@@ -1,634 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.connector.source.Boundedness;
-import org.apache.flink.api.connector.source.SplitEnumerator;
-import org.apache.flink.api.connector.source.SplitEnumeratorContext;
-import org.apache.flink.api.connector.source.SplitsAssignment;
-import org.apache.flink.connector.kafka.source.KafkaSourceOptions;
-import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
-import org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriber;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.KafkaAdminClient;
-import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions;
-import org.apache.kafka.clients.admin.ListOffsetsResult;
-import org.apache.kafka.clients.admin.OffsetSpec;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
-import org.apache.kafka.common.TopicPartition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-/** The enumerator class for Kafka source. */
-@Internal
-public class KafkaSourceEnumerator
- implements SplitEnumerator<KafkaPartitionSplit, KafkaSourceEnumState> {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceEnumerator.class);
- private final KafkaSubscriber subscriber;
- private final OffsetsInitializer startingOffsetInitializer;
- private final OffsetsInitializer stoppingOffsetInitializer;
- private final Properties properties;
- private final long partitionDiscoveryIntervalMs;
- private final SplitEnumeratorContext<KafkaPartitionSplit> context;
- private final Boundedness boundedness;
-
- /** Partitions that have been assigned to readers. */
- private final Set<TopicPartition> assignedPartitions;
-
- /**
- * The discovered and initialized partition splits that are waiting for owner reader to be
- * ready.
- */
- private final Map<Integer, Set<KafkaPartitionSplit>> pendingPartitionSplitAssignment;
-
- /** The consumer group id used for this KafkaSource. */
- private final String consumerGroupId;
-
- // Lazily instantiated or mutable fields.
- private AdminClient adminClient;
-
- // This flag will be marked as true if periodically partition discovery is disabled AND the
- // initializing partition discovery has finished.
- private boolean noMoreNewPartitionSplits = false;
-
- public KafkaSourceEnumerator(
- KafkaSubscriber subscriber,
- OffsetsInitializer startingOffsetInitializer,
- OffsetsInitializer stoppingOffsetInitializer,
- Properties properties,
- SplitEnumeratorContext<KafkaPartitionSplit> context,
- Boundedness boundedness) {
- this(
- subscriber,
- startingOffsetInitializer,
- stoppingOffsetInitializer,
- properties,
- context,
- boundedness,
- Collections.emptySet());
- }
-
- public KafkaSourceEnumerator(
- KafkaSubscriber subscriber,
- OffsetsInitializer startingOffsetInitializer,
- OffsetsInitializer stoppingOffsetInitializer,
- Properties properties,
- SplitEnumeratorContext<KafkaPartitionSplit> context,
- Boundedness boundedness,
- Set<TopicPartition> assignedPartitions) {
- this.subscriber = subscriber;
- this.startingOffsetInitializer = startingOffsetInitializer;
- this.stoppingOffsetInitializer = stoppingOffsetInitializer;
- this.properties = properties;
- this.context = context;
- this.boundedness = boundedness;
-
- this.assignedPartitions = new HashSet<>(assignedPartitions);
- this.pendingPartitionSplitAssignment = new HashMap<>();
- this.partitionDiscoveryIntervalMs =
- KafkaSourceOptions.getOption(
- properties,
- KafkaSourceOptions.PARTITION_DISCOVERY_INTERVAL_MS,
- Long::parseLong);
- this.consumerGroupId = properties.getProperty(ConsumerConfig.GROUP_ID_CONFIG);
- }
-
- /**
- * Start the enumerator.
- *
- * <p>Depending on {@link #partitionDiscoveryIntervalMs}, the enumerator will trigger a one-time
- * partition discovery, or schedule a callable for discover partitions periodically.
- *
- * <p>The invoking chain of partition discovery would be:
- *
- * <ol>
- * <li>{@link #getSubscribedTopicPartitions} in worker thread
- * <li>{@link #checkPartitionChanges} in coordinator thread
- * <li>{@link #initializePartitionSplits} in worker thread
- * <li>{@link #handlePartitionSplitChanges} in coordinator thread
- * </ol>
- */
- @Override
- public void start() {
- adminClient = getKafkaAdminClient();
- if (partitionDiscoveryIntervalMs > 0) {
- LOG.info(
- "Starting the KafkaSourceEnumerator for consumer group {} "
- + "with partition discovery interval of {} ms.",
- consumerGroupId,
- partitionDiscoveryIntervalMs);
- context.callAsync(
- this::getSubscribedTopicPartitions,
- this::checkPartitionChanges,
- 0,
- partitionDiscoveryIntervalMs);
- } else {
- LOG.info(
- "Starting the KafkaSourceEnumerator for consumer group {} "
- + "without periodic partition discovery.",
- consumerGroupId);
- context.callAsync(this::getSubscribedTopicPartitions, this::checkPartitionChanges);
- }
- }
-
- @Override
- public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) {
- // the kafka source pushes splits eagerly, rather than act upon split requests
- }
-
- @Override
- public void addSplitsBack(List<KafkaPartitionSplit> splits, int subtaskId) {
- addPartitionSplitChangeToPendingAssignments(splits);
-
- // If the failed subtask has already restarted, we need to assign pending splits to it
- if (context.registeredReaders().containsKey(subtaskId)) {
- assignPendingPartitionSplits(Collections.singleton(subtaskId));
- }
- }
-
- @Override
- public void addReader(int subtaskId) {
- LOG.debug(
- "Adding reader {} to KafkaSourceEnumerator for consumer group {}.",
- subtaskId,
- consumerGroupId);
- assignPendingPartitionSplits(Collections.singleton(subtaskId));
- }
-
- @Override
- public KafkaSourceEnumState snapshotState(long checkpointId) throws Exception {
- return new KafkaSourceEnumState(assignedPartitions);
- }
-
- @Override
- public void close() {
- if (adminClient != null) {
- adminClient.close();
- }
- }
-
- // ----------------- private methods -------------------
-
- /**
- * List subscribed topic partitions on Kafka brokers.
- *
- * <p>NOTE: This method should only be invoked in the worker executor thread, because it
- * requires network I/O with Kafka brokers.
- *
- * @return Set of subscribed {@link TopicPartition}s
- */
- private Set<TopicPartition> getSubscribedTopicPartitions() {
- return subscriber.getSubscribedTopicPartitions(adminClient);
- }
-
- /**
- * Check if there's any partition changes within subscribed topic partitions fetched by worker
- * thread, and invoke {@link KafkaSourceEnumerator#initializePartitionSplits(PartitionChange)}
- * in worker thread to initialize splits for new partitions.
- *
- * <p>NOTE: This method should only be invoked in the coordinator executor thread.
- *
- * @param fetchedPartitions Map from topic name to its description
- * @param t Exception in worker thread
- */
- private void checkPartitionChanges(Set<TopicPartition> fetchedPartitions, Throwable t) {
- if (t != null) {
- throw new FlinkRuntimeException(
- "Failed to list subscribed topic partitions due to ", t);
- }
- final PartitionChange partitionChange = getPartitionChange(fetchedPartitions);
- if (partitionChange.isEmpty()) {
- return;
- }
- context.callAsync(
- () -> initializePartitionSplits(partitionChange),
- this::handlePartitionSplitChanges);
- }
-
- /**
- * Initialize splits for newly discovered partitions.
- *
- * <p>Enumerator will be responsible for fetching offsets when initializing splits if:
- *
- * <ul>
- * <li>using timestamp for initializing offset
- * <li>or using specified offset, but the offset is not provided for the newly discovered
- * partitions
- * </ul>
- *
- * <p>Otherwise offsets will be initialized by readers.
- *
- * <p>NOTE: This method should only be invoked in the worker executor thread, because it
- * potentially requires network I/O with Kafka brokers for fetching offsets.
- *
- * @param partitionChange Newly discovered and removed partitions
- * @return {@link KafkaPartitionSplit} of new partitions and {@link TopicPartition} of removed
- * partitions
- */
- private PartitionSplitChange initializePartitionSplits(PartitionChange partitionChange) {
- Set<TopicPartition> newPartitions =
- Collections.unmodifiableSet(partitionChange.getNewPartitions());
- OffsetsInitializer.PartitionOffsetsRetriever offsetsRetriever = getOffsetsRetriever();
-
- Map<TopicPartition, Long> startingOffsets =
- startingOffsetInitializer.getPartitionOffsets(newPartitions, offsetsRetriever);
- Map<TopicPartition, Long> stoppingOffsets =
- stoppingOffsetInitializer.getPartitionOffsets(newPartitions, offsetsRetriever);
-
- Set<KafkaPartitionSplit> partitionSplits = new HashSet<>(newPartitions.size());
- for (TopicPartition tp : newPartitions) {
- Long startingOffset = startingOffsets.get(tp);
- long stoppingOffset =
- stoppingOffsets.getOrDefault(tp, KafkaPartitionSplit.NO_STOPPING_OFFSET);
- partitionSplits.add(new KafkaPartitionSplit(tp, startingOffset, stoppingOffset));
- }
- return new PartitionSplitChange(partitionSplits, partitionChange.getRemovedPartitions());
- }
-
- /**
- * Mark partition splits initialized by {@link
- * KafkaSourceEnumerator#initializePartitionSplits(PartitionChange)} as pending and try to
- * assign pending splits to registered readers.
- *
- * <p>NOTE: This method should only be invoked in the coordinator executor thread.
- *
- * @param partitionSplitChange Partition split changes
- * @param t Exception in worker thread
- */
- private void handlePartitionSplitChanges(
- PartitionSplitChange partitionSplitChange, Throwable t) {
- if (t != null) {
- throw new FlinkRuntimeException("Failed to initialize partition splits due to ", t);
- }
- if (partitionDiscoveryIntervalMs < 0) {
- LOG.debug("Partition discovery is disabled.");
- noMoreNewPartitionSplits = true;
- }
- // TODO: Handle removed partitions.
- addPartitionSplitChangeToPendingAssignments(partitionSplitChange.newPartitionSplits);
- assignPendingPartitionSplits(context.registeredReaders().keySet());
- }
-
- // This method should only be invoked in the coordinator executor thread.
- private void addPartitionSplitChangeToPendingAssignments(
- Collection<KafkaPartitionSplit> newPartitionSplits) {
- int numReaders = context.currentParallelism();
- for (KafkaPartitionSplit split : newPartitionSplits) {
- int ownerReader = getSplitOwner(split.getTopicPartition(), numReaders);
- pendingPartitionSplitAssignment
- .computeIfAbsent(ownerReader, r -> new HashSet<>())
- .add(split);
- }
- LOG.debug(
- "Assigned {} to {} readers of consumer group {}.",
- newPartitionSplits,
- numReaders,
- consumerGroupId);
- }
-
- // This method should only be invoked in the coordinator executor thread.
- private void assignPendingPartitionSplits(Set<Integer> pendingReaders) {
- Map<Integer, List<KafkaPartitionSplit>> incrementalAssignment = new HashMap<>();
-
- // Check if there's any pending splits for given readers
- for (int pendingReader : pendingReaders) {
- checkReaderRegistered(pendingReader);
-
- // Remove pending assignment for the reader
- final Set<KafkaPartitionSplit> pendingAssignmentForReader =
- pendingPartitionSplitAssignment.remove(pendingReader);
-
- if (pendingAssignmentForReader != null && !pendingAssignmentForReader.isEmpty()) {
- // Put pending assignment into incremental assignment
- incrementalAssignment
- .computeIfAbsent(pendingReader, (ignored) -> new ArrayList<>())
- .addAll(pendingAssignmentForReader);
-
- // Mark pending partitions as already assigned
- pendingAssignmentForReader.forEach(
- split -> assignedPartitions.add(split.getTopicPartition()));
- }
- }
-
- // Assign pending splits to readers
- if (!incrementalAssignment.isEmpty()) {
- LOG.info("Assigning splits to readers {}", incrementalAssignment);
- context.assignSplits(new SplitsAssignment<>(incrementalAssignment));
- }
-
- // If periodically partition discovery is disabled and the initializing discovery has done,
- // signal NoMoreSplitsEvent to pending readers
- if (noMoreNewPartitionSplits && boundedness == Boundedness.BOUNDED) {
- LOG.debug(
- "No more KafkaPartitionSplits to assign. Sending NoMoreSplitsEvent to reader {}"
- + " in consumer group {}.",
- pendingReaders,
- consumerGroupId);
- pendingReaders.forEach(context::signalNoMoreSplits);
- }
- }
-
- private void checkReaderRegistered(int readerId) {
- if (!context.registeredReaders().containsKey(readerId)) {
- throw new IllegalStateException(
- String.format("Reader %d is not registered to source coordinator", readerId));
- }
- }
-
- @VisibleForTesting
- PartitionChange getPartitionChange(Set<TopicPartition> fetchedPartitions) {
- final Set<TopicPartition> removedPartitions = new HashSet<>();
- Consumer<TopicPartition> dedupOrMarkAsRemoved =
- (tp) -> {
- if (!fetchedPartitions.remove(tp)) {
- removedPartitions.add(tp);
- }
- };
-
- assignedPartitions.forEach(dedupOrMarkAsRemoved);
- pendingPartitionSplitAssignment.forEach(
- (reader, splits) ->
- splits.forEach(
- split -> dedupOrMarkAsRemoved.accept(split.getTopicPartition())));
-
- if (!fetchedPartitions.isEmpty()) {
- LOG.info("Discovered new partitions: {}", fetchedPartitions);
- }
- if (!removedPartitions.isEmpty()) {
- LOG.info("Discovered removed partitions: {}", removedPartitions);
- }
-
- return new PartitionChange(fetchedPartitions, removedPartitions);
- }
-
- private AdminClient getKafkaAdminClient() {
- Properties adminClientProps = new Properties();
- deepCopyProperties(properties, adminClientProps);
- // set client id prefix
- String clientIdPrefix =
- adminClientProps.getProperty(KafkaSourceOptions.CLIENT_ID_PREFIX.key());
- adminClientProps.setProperty(
- ConsumerConfig.CLIENT_ID_CONFIG, clientIdPrefix + "-enumerator-admin-client");
- return AdminClient.create(adminClientProps);
- }
-
- private OffsetsInitializer.PartitionOffsetsRetriever getOffsetsRetriever() {
- String groupId = properties.getProperty(ConsumerConfig.GROUP_ID_CONFIG);
- return new PartitionOffsetsRetrieverImpl(adminClient, groupId);
- }
-
- /**
- * Returns the index of the target subtask that a specific Kafka partition should be assigned
- * to.
- *
- * <p>The resulting distribution of partitions of a single topic has the following contract:
- *
- * <ul>
- * <li>1. Uniformly distributed across subtasks
- * <li>2. Partitions are round-robin distributed (strictly clockwise w.r.t. ascending subtask
- * indices) by using the partition id as the offset from a starting index (i.e., the index
- * of the subtask which partition 0 of the topic will be assigned to, determined using the
- * topic name).
- * </ul>
- *
- * @param tp the Kafka partition to assign.
- * @param numReaders the total number of readers.
- * @return the id of the subtask that owns the split.
- */
- @VisibleForTesting
- static int getSplitOwner(TopicPartition tp, int numReaders) {
- int startIndex = ((tp.topic().hashCode() * 31) & 0x7FFFFFFF) % numReaders;
-
- // here, the assumption is that the id of Kafka partitions are always ascending
- // starting from 0, and therefore can be used directly as the offset clockwise from the
- // start index
- return (startIndex + tp.partition()) % numReaders;
- }
-
- @VisibleForTesting
- static void deepCopyProperties(Properties from, Properties to) {
- for (String key : from.stringPropertyNames()) {
- to.setProperty(key, from.getProperty(key));
- }
- }
-
- // --------------- private class ---------------
-
- /** A container class to hold the newly added partitions and removed partitions. */
- @VisibleForTesting
- static class PartitionChange {
- private final Set<TopicPartition> newPartitions;
- private final Set<TopicPartition> removedPartitions;
-
- PartitionChange(Set<TopicPartition> newPartitions, Set<TopicPartition> removedPartitions) {
- this.newPartitions = newPartitions;
- this.removedPartitions = removedPartitions;
- }
-
- public Set<TopicPartition> getNewPartitions() {
- return newPartitions;
- }
-
- public Set<TopicPartition> getRemovedPartitions() {
- return removedPartitions;
- }
-
- public boolean isEmpty() {
- return newPartitions.isEmpty() && removedPartitions.isEmpty();
- }
- }
-
- private static class PartitionSplitChange {
- private final Set<KafkaPartitionSplit> newPartitionSplits;
- private final Set<TopicPartition> removedPartitions;
-
- private PartitionSplitChange(
- Set<KafkaPartitionSplit> newPartitionSplits,
- Set<TopicPartition> removedPartitions) {
- this.newPartitionSplits = Collections.unmodifiableSet(newPartitionSplits);
- this.removedPartitions = Collections.unmodifiableSet(removedPartitions);
- }
- }
-
- /** The implementation for offsets retriever with a consumer and an admin client. */
- @VisibleForTesting
- public static class PartitionOffsetsRetrieverImpl
- implements OffsetsInitializer.PartitionOffsetsRetriever, AutoCloseable {
- private final AdminClient adminClient;
- private final String groupId;
-
- public PartitionOffsetsRetrieverImpl(AdminClient adminClient, String groupId) {
- this.adminClient = adminClient;
- this.groupId = groupId;
- }
-
- @Override
- public Map<TopicPartition, Long> committedOffsets(Collection<TopicPartition> partitions) {
- ListConsumerGroupOffsetsOptions options =
- new ListConsumerGroupOffsetsOptions()
- .topicPartitions(new ArrayList<>(partitions));
- try {
- return adminClient
- .listConsumerGroupOffsets(groupId, options)
- .partitionsToOffsetAndMetadata()
- .thenApply(
- result -> {
- Map<TopicPartition, Long> offsets = new HashMap<>();
- result.forEach(
- (tp, oam) -> {
- if (oam != null) {
- offsets.put(tp, oam.offset());
- }
- });
- return offsets;
- })
- .get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new FlinkRuntimeException(
- "Interrupted while listing offsets for consumer group " + groupId, e);
- } catch (ExecutionException e) {
- throw new FlinkRuntimeException(
- "Failed to fetch committed offsets for consumer group "
- + groupId
- + " due to",
- e);
- }
- }
-
- /**
- * List offsets for the specified partitions and OffsetSpec. This operation enables to find
- * the beginning offset, end offset as well as the offset matching a timestamp in
- * partitions.
- *
- * @see KafkaAdminClient#listOffsets(Map)
- * @param topicPartitionOffsets The mapping from partition to the OffsetSpec to look up.
- * @return The list offsets result.
- */
- private Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> listOffsets(
- Map<TopicPartition, OffsetSpec> topicPartitionOffsets) {
- try {
- return adminClient
- .listOffsets(topicPartitionOffsets)
- .all()
- .thenApply(
- result -> {
- Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo>
- offsets = new HashMap<>();
- result.forEach(
- (tp, listOffsetsResultInfo) -> {
- if (listOffsetsResultInfo != null) {
- offsets.put(tp, listOffsetsResultInfo);
- }
- });
- return offsets;
- })
- .get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new FlinkRuntimeException(
- "Interrupted while listing offsets for topic partitions: "
- + topicPartitionOffsets,
- e);
- } catch (ExecutionException e) {
- throw new FlinkRuntimeException(
- "Failed to list offsets for topic partitions: "
- + topicPartitionOffsets
- + " due to",
- e);
- }
- }
-
- private Map<TopicPartition, Long> listOffsets(
- Collection<TopicPartition> partitions, OffsetSpec offsetSpec) {
- return listOffsets(
- partitions.stream()
- .collect(
- Collectors.toMap(
- partition -> partition, __ -> offsetSpec)))
- .entrySet().stream()
- .collect(
- Collectors.toMap(
- Map.Entry::getKey, entry -> entry.getValue().offset()));
- }
-
- @Override
- public Map<TopicPartition, Long> endOffsets(Collection<TopicPartition> partitions) {
- return listOffsets(partitions, OffsetSpec.latest());
- }
-
- @Override
- public Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition> partitions) {
- return listOffsets(partitions, OffsetSpec.earliest());
- }
-
- @Override
- public Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
- Map<TopicPartition, Long> timestampsToSearch) {
- return listOffsets(
- timestampsToSearch.entrySet().stream()
- .collect(
- Collectors.toMap(
- Map.Entry::getKey,
- entry ->
- OffsetSpec.forTimestamp(
- entry.getValue()))))
- .entrySet().stream()
- // OffsetAndTimestamp cannot be initialized with a negative offset, which is
- // possible if the timestamp does not correspond to an offset and the topic
- // partition is empty
- .filter(entry -> entry.getValue().offset() >= 0)
- .collect(
- Collectors.toMap(
- Map.Entry::getKey,
- entry ->
- new OffsetAndTimestamp(
- entry.getValue().offset(),
- entry.getValue().timestamp(),
- entry.getValue().leaderEpoch())));
- }
-
- @Override
- public void close() throws Exception {
- adminClient.close(Duration.ZERO);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/NoStoppingOffsetsInitializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/NoStoppingOffsetsInitializer.java
deleted file mode 100644
index bb94d1666c4..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/NoStoppingOffsetsInitializer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.flink.annotation.Internal;
-
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-/**
- * An implementation of {@link OffsetsInitializer} which does not initialize anything.
- *
- * <p>This class is used as the default stopping offsets initializer for unbounded Kafka sources.
- */
-@Internal
-public class NoStoppingOffsetsInitializer implements OffsetsInitializer {
- private static final long serialVersionUID = 4186323669290142732L;
-
- @Override
- public Map<TopicPartition, Long> getPartitionOffsets(
- Collection<TopicPartition> partitions,
- PartitionOffsetsRetriever partitionOffsetsRetriever) {
- return Collections.emptyMap();
- }
-
- @Override
- public OffsetResetStrategy getAutoOffsetResetStrategy() {
- throw new UnsupportedOperationException(
- "The NoStoppingOffsetsInitializer does not have an OffsetResetStrategy. It should only be used "
- + "to end offset.");
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializer.java
deleted file mode 100644
index db682c6b085..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializer.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.kafka.source.KafkaSource;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-
-import org.apache.kafka.clients.admin.KafkaAdminClient;
-import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions;
-import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Map;
-
-/**
- * An interface for users to specify the starting / stopping offset of a {@link
- * KafkaPartitionSplit}.
- *
- * @see ReaderHandledOffsetsInitializer
- * @see SpecifiedOffsetsInitializer
- * @see TimestampOffsetsInitializer
- */
-@PublicEvolving
-public interface OffsetsInitializer extends Serializable {
-
- /**
- * Get the initial offsets for the given Kafka partitions. These offsets will be used as either
- * starting offsets or stopping offsets of the Kafka partitions.
- *
- * <p>If the implementation returns a starting offset which causes {@code
- * OffsetsOutOfRangeException} from Kafka. The {@link OffsetResetStrategy} provided by the
- * {@link #getAutoOffsetResetStrategy()} will be used to reset the offset.
- *
- * @param partitions the Kafka partitions to get the starting offsets.
- * @param partitionOffsetsRetriever a helper to retrieve information of the Kafka partitions.
- * @return A mapping from Kafka partition to their offsets to start consuming from.
- */
- Map<TopicPartition, Long> getPartitionOffsets(
- Collection<TopicPartition> partitions,
- PartitionOffsetsRetriever partitionOffsetsRetriever);
-
- /**
- * Get the auto offset reset strategy in case the initialized offsets falls out of the range.
- *
- * <p>The OffsetStrategy is only used when the offset initializer is used to initialize the
- * starting offsets and the starting offsets is out of range.
- *
- * @return An {@link OffsetResetStrategy} to use if the initialized offsets are out of the
- * range.
- */
- OffsetResetStrategy getAutoOffsetResetStrategy();
-
- /**
- * An interface that provides necessary information to the {@link OffsetsInitializer} to get the
- * initial offsets of the Kafka partitions.
- */
- interface PartitionOffsetsRetriever {
-
- /**
- * The group id should be the set for {@link KafkaSource KafkaSource} before invoking this
- * method. Otherwise an {@code IllegalStateException} will be thrown.
- *
- * @see KafkaAdminClient#listConsumerGroupOffsets(String, ListConsumerGroupOffsetsOptions)
- * @throws IllegalStateException if the group id is not set for the {@code KafkaSource}.
- */
- Map<TopicPartition, Long> committedOffsets(Collection<TopicPartition> partitions);
-
- /** List end offsets for the specified partitions. */
- Map<TopicPartition, Long> endOffsets(Collection<TopicPartition> partitions);
-
- /** List beginning offsets for the specified partitions. */
- Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition> partitions);
-
- /** List offsets matching a timestamp for the specified partitions. */
- Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
- Map<TopicPartition, Long> timestampsToSearch);
- }
-
- // --------------- factory methods ---------------
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the committed offsets. An
- * exception will be thrown at runtime if there is no committed offsets.
- *
- * @return an offset initializer which initialize the offsets to the committed offsets.
- */
- static OffsetsInitializer committedOffsets() {
- return committedOffsets(OffsetResetStrategy.NONE);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the committed offsets. Use
- * the given {@link OffsetResetStrategy} to initialize the offsets if the committed offsets does
- * not exist.
- *
- * @param offsetResetStrategy the offset reset strategy to use when the committed offsets do not
- * exist.
- * @return an {@link OffsetsInitializer} which initializes the offsets to the committed offsets.
- */
- static OffsetsInitializer committedOffsets(OffsetResetStrategy offsetResetStrategy) {
- return new ReaderHandledOffsetsInitializer(
- KafkaPartitionSplit.COMMITTED_OFFSET, offsetResetStrategy);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets in each partition so that the
- * initialized offset is the offset of the first record whose record timestamp is greater than
- * or equals the given timestamp (milliseconds).
- *
- * @param timestamp the timestamp (milliseconds) to start the consumption.
- * @return an {@link OffsetsInitializer} which initializes the offsets based on the given
- * timestamp.
- * @see KafkaAdminClient#listOffsets(Map)
- */
- static OffsetsInitializer timestamp(long timestamp) {
- return new TimestampOffsetsInitializer(timestamp);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the earliest available
- * offsets of each partition.
- *
- * @return an {@link OffsetsInitializer} which initializes the offsets to the earliest available
- * offsets.
- */
- static OffsetsInitializer earliest() {
- return new ReaderHandledOffsetsInitializer(
- KafkaPartitionSplit.EARLIEST_OFFSET, OffsetResetStrategy.EARLIEST);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the latest offsets of each
- * partition.
- *
- * @return an {@link OffsetsInitializer} which initializes the offsets to the latest offsets.
- */
- static OffsetsInitializer latest() {
- return new ReaderHandledOffsetsInitializer(
- KafkaPartitionSplit.LATEST_OFFSET, OffsetResetStrategy.LATEST);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the specified offsets.
- *
- * @param offsets the specified offsets for each partition.
- * @return an {@link OffsetsInitializer} which initializes the offsets to the specified offsets.
- */
- static OffsetsInitializer offsets(Map<TopicPartition, Long> offsets) {
- return new SpecifiedOffsetsInitializer(offsets, OffsetResetStrategy.EARLIEST);
- }
-
- /**
- * Get an {@link OffsetsInitializer} which initializes the offsets to the specified offsets. Use
- * the given {@link OffsetResetStrategy} to initialize the offsets in case the specified offset
- * is out of range.
- *
- * @param offsets the specified offsets for each partition.
- * @param offsetResetStrategy the {@link OffsetResetStrategy} to use when the specified offset
- * is out of range.
- * @return an {@link OffsetsInitializer} which initializes the offsets to the specified offsets.
- */
- static OffsetsInitializer offsets(
- Map<TopicPartition, Long> offsets, OffsetResetStrategy offsetResetStrategy) {
- return new SpecifiedOffsetsInitializer(offsets, offsetResetStrategy);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializerValidator.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializerValidator.java
deleted file mode 100644
index c1981079ba1..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/OffsetsInitializerValidator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.flink.annotation.Internal;
-
-import java.util.Properties;
-
-/**
- * Interface for validating {@link OffsetsInitializer} with properties from {@link
- * org.apache.flink.connector.kafka.source.KafkaSource}.
- */
-@Internal
-public interface OffsetsInitializerValidator {
-
- /**
- * Validate offsets initializer with properties of Kafka source.
- *
- * @param kafkaSourceProperties Properties of Kafka source
- * @throws IllegalStateException if validation fails
- */
- void validate(Properties kafkaSourceProperties) throws IllegalStateException;
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/ReaderHandledOffsetsInitializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/ReaderHandledOffsetsInitializer.java
deleted file mode 100644
index 026320d9392..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/ReaderHandledOffsetsInitializer.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * A initializer that initialize the partitions to the earliest / latest / last-committed offsets.
- * The offsets initialization are taken care of by the {@code KafkaPartitionSplitReader} instead of
- * by the {@code KafkaSourceEnumerator}.
- *
- * <p>Package private and should be instantiated via {@link OffsetsInitializer}.
- */
-class ReaderHandledOffsetsInitializer implements OffsetsInitializer, OffsetsInitializerValidator {
- private static final long serialVersionUID = 172938052008787981L;
- private final long startingOffset;
- private final OffsetResetStrategy offsetResetStrategy;
-
- /**
- * The only valid value for startingOffset is following. {@link
- * KafkaPartitionSplit#EARLIEST_OFFSET EARLIEST_OFFSET}, {@link
- * KafkaPartitionSplit#LATEST_OFFSET LATEST_OFFSET}, {@link KafkaPartitionSplit#COMMITTED_OFFSET
- * COMMITTED_OFFSET}
- */
- ReaderHandledOffsetsInitializer(long startingOffset, OffsetResetStrategy offsetResetStrategy) {
- this.startingOffset = startingOffset;
- this.offsetResetStrategy = offsetResetStrategy;
- }
-
- @Override
- public Map<TopicPartition, Long> getPartitionOffsets(
- Collection<TopicPartition> partitions,
- PartitionOffsetsRetriever partitionOffsetsRetriever) {
- Map<TopicPartition, Long> initialOffsets = new HashMap<>();
- for (TopicPartition tp : partitions) {
- initialOffsets.put(tp, startingOffset);
- }
- return initialOffsets;
- }
-
- @Override
- public OffsetResetStrategy getAutoOffsetResetStrategy() {
- return offsetResetStrategy;
- }
-
- @Override
- public void validate(Properties kafkaSourceProperties) {
- if (startingOffset == KafkaPartitionSplit.COMMITTED_OFFSET) {
- checkState(
- kafkaSourceProperties.containsKey(ConsumerConfig.GROUP_ID_CONFIG),
- String.format(
- "Property %s is required when using committed offset for offsets initializer",
- ConsumerConfig.GROUP_ID_CONFIG));
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/SpecifiedOffsetsInitializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/SpecifiedOffsetsInitializer.java
deleted file mode 100644
index 5766a5fc155..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/SpecifiedOffsetsInitializer.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * An implementation of {@link OffsetsInitializer} which initializes the offsets of the partition
- * according to the user specified offsets.
- *
- * <p>Package private and should be instantiated via {@link OffsetsInitializer}.
- */
-class SpecifiedOffsetsInitializer implements OffsetsInitializer, OffsetsInitializerValidator {
- private static final long serialVersionUID = 1649702397250402877L;
- private final Map<TopicPartition, Long> initialOffsets;
- private final OffsetResetStrategy offsetResetStrategy;
-
- SpecifiedOffsetsInitializer(
- Map<TopicPartition, Long> initialOffsets, OffsetResetStrategy offsetResetStrategy) {
- this.initialOffsets = Collections.unmodifiableMap(initialOffsets);
- this.offsetResetStrategy = offsetResetStrategy;
- }
-
- @Override
- public Map<TopicPartition, Long> getPartitionOffsets(
- Collection<TopicPartition> partitions,
- PartitionOffsetsRetriever partitionOffsetsRetriever) {
- Map<TopicPartition, Long> offsets = new HashMap<>();
- List<TopicPartition> toLookup = new ArrayList<>();
- for (TopicPartition tp : partitions) {
- Long offset = initialOffsets.get(tp);
- if (offset == null) {
- toLookup.add(tp);
- } else {
- offsets.put(tp, offset);
- }
- }
- if (!toLookup.isEmpty()) {
- // First check the committed offsets.
- Map<TopicPartition, Long> committedOffsets =
- partitionOffsetsRetriever.committedOffsets(toLookup);
- offsets.putAll(committedOffsets);
- toLookup.removeAll(committedOffsets.keySet());
-
- switch (offsetResetStrategy) {
- case EARLIEST:
- offsets.putAll(partitionOffsetsRetriever.beginningOffsets(toLookup));
- break;
- case LATEST:
- offsets.putAll(partitionOffsetsRetriever.endOffsets(toLookup));
- break;
- default:
- throw new IllegalStateException(
- "Cannot find initial offsets for partitions: " + toLookup);
- }
- }
- return offsets;
- }
-
- @Override
- public OffsetResetStrategy getAutoOffsetResetStrategy() {
- return offsetResetStrategy;
- }
-
- @Override
- public void validate(Properties kafkaSourceProperties) {
- initialOffsets.forEach(
- (tp, offset) -> {
- if (offset == KafkaPartitionSplit.COMMITTED_OFFSET) {
- checkState(
- kafkaSourceProperties.containsKey(ConsumerConfig.GROUP_ID_CONFIG),
- String.format(
- "Property %s is required because partition %s is initialized with committed offset",
- ConsumerConfig.GROUP_ID_CONFIG, tp));
- }
- });
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/TimestampOffsetsInitializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/TimestampOffsetsInitializer.java
deleted file mode 100644
index f411307f6c5..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/initializer/TimestampOffsetsInitializer.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.initializer;
-
-import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * An implementation of {@link OffsetsInitializer} to initialize the offsets based on a timestamp.
- *
- * <p>Package private and should be instantiated via {@link OffsetsInitializer}.
- */
-class TimestampOffsetsInitializer implements OffsetsInitializer {
- private static final long serialVersionUID = 2932230571773627233L;
- private final long startingTimestamp;
-
- TimestampOffsetsInitializer(long startingTimestamp) {
- this.startingTimestamp = startingTimestamp;
- }
-
- @Override
- public Map<TopicPartition, Long> getPartitionOffsets(
- Collection<TopicPartition> partitions,
- PartitionOffsetsRetriever partitionOffsetsRetriever) {
- Map<TopicPartition, Long> startingTimestamps = new HashMap<>();
- Map<TopicPartition, Long> initialOffsets = new HashMap<>();
-
- // First get the current end offsets of the partitions. This is going to be used
- // in case we cannot find a suitable offsets based on the timestamp, i.e. the message
- // meeting the requirement of the timestamp have not been produced to Kafka yet, in
- // this case, we just use the latest offset.
- // We need to get the latest offsets before querying offsets by time to ensure that
- // no message is going to be missed.
- Map<TopicPartition, Long> endOffsets = partitionOffsetsRetriever.endOffsets(partitions);
- partitions.forEach(tp -> startingTimestamps.put(tp, startingTimestamp));
- Map<TopicPartition, OffsetAndTimestamp> topicPartitionOffsetAndTimestampMap =
- partitionOffsetsRetriever.offsetsForTimes(startingTimestamps);
-
- for (TopicPartition tp : partitions) {
- // offset may not have been resolved
- if (topicPartitionOffsetAndTimestampMap.containsKey(tp)) {
- initialOffsets.put(tp, topicPartitionOffsetAndTimestampMap.get(tp).offset());
- } else {
- initialOffsets.put(tp, endOffsets.get(tp));
- }
- }
-
- return initialOffsets;
- }
-
- @Override
- public OffsetResetStrategy getAutoOffsetResetStrategy() {
- return OffsetResetStrategy.LATEST;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriber.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriber.java
deleted file mode 100644
index 1b819fb2305..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriber.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.subscriber;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.common.TopicPartition;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Kafka consumer allows a few different ways to consume from the topics, including:
- *
- * <ol>
- * <li>Subscribe from a collection of topics.
- * <li>Subscribe to a topic pattern using Java {@code Regex}.
- * <li>Assign specific partitions.
- * </ol>
- *
- * <p>The KafkaSubscriber provides a unified interface for the Kafka source to support all these
- * three types of subscribing mode.
- */
-@PublicEvolving
-public interface KafkaSubscriber extends Serializable {
-
- /**
- * Get a set of subscribed {@link TopicPartition}s.
- *
- * @param adminClient The admin client used to retrieve subscribed topic partitions.
- * @return A set of subscribed {@link TopicPartition}s
- */
- Set<TopicPartition> getSubscribedTopicPartitions(AdminClient adminClient);
-
- // ----------------- factory methods --------------
-
- static KafkaSubscriber getTopicListSubscriber(List<String> topics) {
- return new TopicListSubscriber(topics);
- }
-
- static KafkaSubscriber getTopicPatternSubscriber(Pattern topicPattern) {
- return new TopicPatternSubscriber(topicPattern);
- }
-
- static KafkaSubscriber getPartitionSetSubscriber(Set<TopicPartition> partitions) {
- return new PartitionSetSubscriber(partitions);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriberUtils.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriberUtils.java
deleted file mode 100644
index 404ffaef863..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/KafkaSubscriberUtils.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.subscriber;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.TopicDescription;
-
-import java.util.Map;
-import java.util.Set;
-
-/** The base implementations of {@link KafkaSubscriber}. */
-class KafkaSubscriberUtils {
-
- private KafkaSubscriberUtils() {}
-
- static Map<String, TopicDescription> getAllTopicMetadata(AdminClient adminClient) {
- try {
- Set<String> allTopicNames = adminClient.listTopics().names().get();
- return getTopicMetadata(adminClient, allTopicNames);
- } catch (Exception e) {
- throw new RuntimeException("Failed to get metadata for all topics.", e);
- }
- }
-
- static Map<String, TopicDescription> getTopicMetadata(
- AdminClient adminClient, Set<String> topicNames) {
- try {
- return adminClient.describeTopics(topicNames).allTopicNames().get();
- } catch (Exception e) {
- throw new RuntimeException(
- String.format("Failed to get metadata for topics %s.", topicNames), e);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/PartitionSetSubscriber.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/PartitionSetSubscriber.java
deleted file mode 100644
index 3423b0f909e..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/PartitionSetSubscriber.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.subscriber;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.TopicDescription;
-import org.apache.kafka.common.TopicPartition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriberUtils.getTopicMetadata;
-
-/** A subscriber for a partition set. */
-class PartitionSetSubscriber implements KafkaSubscriber {
- private static final long serialVersionUID = 390970375272146036L;
- private static final Logger LOG = LoggerFactory.getLogger(PartitionSetSubscriber.class);
- private final Set<TopicPartition> subscribedPartitions;
-
- PartitionSetSubscriber(Set<TopicPartition> partitions) {
- this.subscribedPartitions = partitions;
- }
-
- @Override
- public Set<TopicPartition> getSubscribedTopicPartitions(AdminClient adminClient) {
- final Set<String> topicNames =
- subscribedPartitions.stream()
- .map(TopicPartition::topic)
- .collect(Collectors.toSet());
-
- LOG.debug("Fetching descriptions for topics: {}", topicNames);
- final Map<String, TopicDescription> topicMetadata =
- getTopicMetadata(adminClient, topicNames);
-
- Set<TopicPartition> existingSubscribedPartitions = new HashSet<>();
-
- for (TopicPartition subscribedPartition : this.subscribedPartitions) {
- if (topicMetadata.containsKey(subscribedPartition.topic())
- && partitionExistsInTopic(
- subscribedPartition, topicMetadata.get(subscribedPartition.topic()))) {
- existingSubscribedPartitions.add(subscribedPartition);
- } else {
- throw new RuntimeException(
- String.format(
- "Partition '%s' does not exist on Kafka brokers",
- subscribedPartition));
- }
- }
-
- return existingSubscribedPartitions;
- }
-
- private boolean partitionExistsInTopic(TopicPartition partition, TopicDescription topic) {
- return topic.partitions().size() > partition.partition();
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicListSubscriber.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicListSubscriber.java
deleted file mode 100644
index b2ad844ab9d..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicListSubscriber.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.subscriber;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.TopicDescription;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.TopicPartitionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriberUtils.getTopicMetadata;
-
-/**
- * A subscriber to a fixed list of topics. The subscribed topics must have existed in the Kafka
- * cluster, otherwise an exception will be thrown.
- */
-class TopicListSubscriber implements KafkaSubscriber {
- private static final long serialVersionUID = -6917603843104947866L;
- private static final Logger LOG = LoggerFactory.getLogger(TopicListSubscriber.class);
- private final List<String> topics;
-
- TopicListSubscriber(List<String> topics) {
- this.topics = topics;
- }
-
- @Override
- public Set<TopicPartition> getSubscribedTopicPartitions(AdminClient adminClient) {
- LOG.debug("Fetching descriptions for topics: {}", topics);
- final Map<String, TopicDescription> topicMetadata =
- getTopicMetadata(adminClient, new HashSet<>(topics));
-
- Set<TopicPartition> subscribedPartitions = new HashSet<>();
- for (TopicDescription topic : topicMetadata.values()) {
- for (TopicPartitionInfo partition : topic.partitions()) {
- subscribedPartitions.add(new TopicPartition(topic.name(), partition.partition()));
- }
- }
-
- return subscribedPartitions;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicPatternSubscriber.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicPatternSubscriber.java
deleted file mode 100644
index 2a9a753316d..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/enumerator/subscriber/TopicPatternSubscriber.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.enumerator.subscriber;
-
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.TopicDescription;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.TopicPartitionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import static org.apache.flink.connector.kafka.source.enumerator.subscriber.KafkaSubscriberUtils.getAllTopicMetadata;
-
-/** A subscriber to a topic pattern. */
-class TopicPatternSubscriber implements KafkaSubscriber {
- private static final long serialVersionUID = -7471048577725467797L;
- private static final Logger LOG = LoggerFactory.getLogger(TopicPatternSubscriber.class);
- private final Pattern topicPattern;
-
- TopicPatternSubscriber(Pattern topicPattern) {
- this.topicPattern = topicPattern;
- }
-
- @Override
- public Set<TopicPartition> getSubscribedTopicPartitions(AdminClient adminClient) {
- LOG.debug("Fetching descriptions for all topics on Kafka cluster");
- final Map<String, TopicDescription> allTopicMetadata = getAllTopicMetadata(adminClient);
-
- Set<TopicPartition> subscribedTopicPartitions = new HashSet<>();
-
- allTopicMetadata.forEach(
- (topicName, topicDescription) -> {
- if (topicPattern.matcher(topicName).matches()) {
- for (TopicPartitionInfo partition : topicDescription.partitions()) {
- subscribedTopicPartitions.add(
- new TopicPartition(
- topicDescription.name(), partition.partition()));
- }
- }
- });
-
- return subscribedTopicPartitions;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/metrics/KafkaSourceReaderMetrics.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/metrics/KafkaSourceReaderMetrics.java
deleted file mode 100644
index f94243ea864..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/metrics/KafkaSourceReaderMetrics.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.metrics;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.connector.kafka.MetricUtil;
-import org.apache.flink.connector.kafka.source.reader.KafkaSourceReader;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.metrics.groups.OperatorIOMetricGroup;
-import org.apache.flink.metrics.groups.SourceReaderMetricGroup;
-import org.apache.flink.runtime.metrics.MetricNames;
-import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.TopicPartition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.function.Predicate;
-
-/**
- * A collection class for handling metrics in {@link KafkaSourceReader}.
- *
- * <p>All metrics of Kafka source reader are registered under group "KafkaSourceReader", which is a
- * child group of {@link org.apache.flink.metrics.groups.OperatorMetricGroup}. Metrics related to a
- * specific topic partition will be registered in the group
- * "KafkaSourceReader.topic.{topic_name}.partition.{partition_id}".
- *
- * <p>For example, current consuming offset of topic "my-topic" and partition 1 will be reported in
- * metric:
- * "{some_parent_groups}.operator.KafkaSourceReader.topic.my-topic.partition.1.currentOffset"
- *
- * <p>and number of successful commits will be reported in metric:
- * "{some_parent_groups}.operator.KafkaSourceReader.commitsSucceeded"
- *
- * <p>All metrics of Kafka consumer are also registered under group
- * "KafkaSourceReader.KafkaConsumer". For example, Kafka consumer metric "records-consumed-total"
- * can be found at:
- * {some_parent_groups}.operator.KafkaSourceReader.KafkaConsumer.records-consumed-total"
- */
-@PublicEvolving
-public class KafkaSourceReaderMetrics {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceReaderMetrics.class);
-
- // Constants
- public static final String KAFKA_SOURCE_READER_METRIC_GROUP = "KafkaSourceReader";
- public static final String TOPIC_GROUP = "topic";
- public static final String PARTITION_GROUP = "partition";
- public static final String CURRENT_OFFSET_METRIC_GAUGE = "currentOffset";
- public static final String COMMITTED_OFFSET_METRIC_GAUGE = "committedOffset";
- public static final String COMMITS_SUCCEEDED_METRIC_COUNTER = "commitsSucceeded";
- public static final String COMMITS_FAILED_METRIC_COUNTER = "commitsFailed";
- public static final String KAFKA_CONSUMER_METRIC_GROUP = "KafkaConsumer";
-
- // Kafka raw metric names and group names
- public static final String CONSUMER_FETCH_MANAGER_GROUP = "consumer-fetch-manager-metrics";
- public static final String BYTES_CONSUMED_TOTAL = "bytes-consumed-total";
- public static final String RECORDS_LAG = "records-lag";
-
- public static final long INITIAL_OFFSET = -1;
-
- // Source reader metric group
- private final SourceReaderMetricGroup sourceReaderMetricGroup;
-
- // Metric group for registering Kafka specific metrics
- private final MetricGroup kafkaSourceReaderMetricGroup;
-
- // Successful / Failed commits counters
- private final Counter commitsSucceeded;
- private final Counter commitsFailed;
-
- // Map for tracking current consuming / committing offsets
- private final Map<TopicPartition, Offset> offsets = new HashMap<>();
-
- // Map for tracking records lag of topic partitions
- @Nullable private ConcurrentMap<TopicPartition, Metric> recordsLagMetrics;
-
- // Kafka raw metric for bytes consumed total
- @Nullable private Metric bytesConsumedTotalMetric;
-
- /** Number of bytes consumed total at the latest {@link #updateNumBytesInCounter()}. */
- private long latestBytesConsumedTotal;
-
- public KafkaSourceReaderMetrics(SourceReaderMetricGroup sourceReaderMetricGroup) {
- this.sourceReaderMetricGroup = sourceReaderMetricGroup;
- this.kafkaSourceReaderMetricGroup =
- sourceReaderMetricGroup.addGroup(KAFKA_SOURCE_READER_METRIC_GROUP);
- this.commitsSucceeded =
- this.kafkaSourceReaderMetricGroup.counter(COMMITS_SUCCEEDED_METRIC_COUNTER);
- this.commitsFailed =
- this.kafkaSourceReaderMetricGroup.counter(COMMITS_FAILED_METRIC_COUNTER);
- }
-
- /**
- * Register metrics of KafkaConsumer in Kafka metric group.
- *
- * @param kafkaConsumer Kafka consumer used by partition split reader.
- */
- @SuppressWarnings("Convert2MethodRef")
- public void registerKafkaConsumerMetrics(KafkaConsumer<?, ?> kafkaConsumer) {
- final Map<MetricName, ? extends Metric> kafkaConsumerMetrics = kafkaConsumer.metrics();
- if (kafkaConsumerMetrics == null) {
- LOG.warn("Consumer implementation does not support metrics");
- return;
- }
-
- final MetricGroup kafkaConsumerMetricGroup =
- kafkaSourceReaderMetricGroup.addGroup(KAFKA_CONSUMER_METRIC_GROUP);
-
- kafkaConsumerMetrics.forEach(
- (name, metric) ->
- kafkaConsumerMetricGroup.gauge(name.name(), () -> metric.metricValue()));
- }
-
- /**
- * Register metric groups for the given {@link TopicPartition}.
- *
- * @param tp Registering topic partition
- */
- public void registerTopicPartition(TopicPartition tp) {
- offsets.put(tp, new Offset(INITIAL_OFFSET, INITIAL_OFFSET));
- registerOffsetMetricsForTopicPartition(tp);
- }
-
- /**
- * Update current consuming offset of the given {@link TopicPartition}.
- *
- * @param tp Updating topic partition
- * @param offset Current consuming offset
- */
- public void recordCurrentOffset(TopicPartition tp, long offset) {
- checkTopicPartitionTracked(tp);
- offsets.get(tp).currentOffset = offset;
- }
-
- /**
- * Update the latest committed offset of the given {@link TopicPartition}.
- *
- * @param tp Updating topic partition
- * @param offset Committing offset
- */
- public void recordCommittedOffset(TopicPartition tp, long offset) {
- checkTopicPartitionTracked(tp);
- offsets.get(tp).committedOffset = offset;
- }
-
- /** Mark a successful commit. */
- public void recordSucceededCommit() {
- commitsSucceeded.inc();
- }
-
- /** Mark a failure commit. */
- public void recordFailedCommit() {
- commitsFailed.inc();
- }
-
- /**
- * Register {@link MetricNames#IO_NUM_BYTES_IN}.
- *
- * @param consumer Kafka consumer
- */
- public void registerNumBytesIn(KafkaConsumer<?, ?> consumer) {
- try {
- Predicate<Map.Entry<MetricName, ? extends Metric>> filter =
- (entry) ->
- entry.getKey().group().equals(CONSUMER_FETCH_MANAGER_GROUP)
- && entry.getKey().name().equals(BYTES_CONSUMED_TOTAL)
- && !entry.getKey().tags().containsKey("topic");
- this.bytesConsumedTotalMetric = MetricUtil.getKafkaMetric(consumer.metrics(), filter);
- } catch (IllegalStateException e) {
- LOG.warn(
- String.format(
- "Error when getting Kafka consumer metric \"%s\". "
- + "I/O metric \"%s\" will not be reported. ",
- BYTES_CONSUMED_TOTAL, MetricNames.IO_NUM_BYTES_IN),
- e);
- }
- }
-
- /**
- * Add a partition's records-lag metric to tracking list if this partition never appears before.
- *
- * <p>This method also lazily register {@link
- * org.apache.flink.runtime.metrics.MetricNames#PENDING_RECORDS} in {@link
- * SourceReaderMetricGroup}
- *
- * @param consumer Kafka consumer
- * @param tp Topic partition
- */
- public void maybeAddRecordsLagMetric(KafkaConsumer<?, ?> consumer, TopicPartition tp) {
- // Lazily register pendingRecords
- if (recordsLagMetrics == null) {
- this.recordsLagMetrics = new ConcurrentHashMap<>();
- this.sourceReaderMetricGroup.setPendingRecordsGauge(
- () -> {
- long pendingRecordsTotal = 0;
- for (Metric recordsLagMetric : this.recordsLagMetrics.values()) {
- pendingRecordsTotal +=
- ((Double) recordsLagMetric.metricValue()).longValue();
- }
- return pendingRecordsTotal;
- });
- }
- recordsLagMetrics.computeIfAbsent(
- tp, (ignored) -> getRecordsLagMetric(consumer.metrics(), tp));
- }
-
- /**
- * Remove a partition's records-lag metric from tracking list.
- *
- * @param tp Unassigned topic partition
- */
- public void removeRecordsLagMetric(TopicPartition tp) {
- if (recordsLagMetrics != null) {
- recordsLagMetrics.remove(tp);
- }
- }
-
- /**
- * Update {@link org.apache.flink.runtime.metrics.MetricNames#IO_NUM_BYTES_IN}.
- *
- * <p>Instead of simply setting {@link OperatorIOMetricGroup#getNumBytesInCounter()} to the same
- * value as bytes-consumed-total from Kafka consumer, which will screw {@link
- * TaskIOMetricGroup#getNumBytesInCounter()} if chained sources exist, we track the increment of
- * bytes-consumed-total and count it towards the counter.
- */
- public void updateNumBytesInCounter() {
- if (this.bytesConsumedTotalMetric != null) {
- long bytesConsumedUntilNow =
- ((Number) this.bytesConsumedTotalMetric.metricValue()).longValue();
- long bytesConsumedSinceLastUpdate = bytesConsumedUntilNow - latestBytesConsumedTotal;
- this.sourceReaderMetricGroup
- .getIOMetricGroup()
- .getNumBytesInCounter()
- .inc(bytesConsumedSinceLastUpdate);
- latestBytesConsumedTotal = bytesConsumedUntilNow;
- }
- }
-
- // -------- Helper functions --------
- private void registerOffsetMetricsForTopicPartition(TopicPartition tp) {
- final MetricGroup topicPartitionGroup =
- this.kafkaSourceReaderMetricGroup
- .addGroup(TOPIC_GROUP, tp.topic())
- .addGroup(PARTITION_GROUP, String.valueOf(tp.partition()));
- topicPartitionGroup.gauge(
- CURRENT_OFFSET_METRIC_GAUGE,
- () ->
- offsets.getOrDefault(tp, new Offset(INITIAL_OFFSET, INITIAL_OFFSET))
- .currentOffset);
- topicPartitionGroup.gauge(
- COMMITTED_OFFSET_METRIC_GAUGE,
- () ->
- offsets.getOrDefault(tp, new Offset(INITIAL_OFFSET, INITIAL_OFFSET))
- .committedOffset);
- }
-
- private void checkTopicPartitionTracked(TopicPartition tp) {
- if (!offsets.containsKey(tp)) {
- throw new IllegalArgumentException(
- String.format("TopicPartition %s is not tracked", tp));
- }
- }
-
- private @Nullable Metric getRecordsLagMetric(
- Map<MetricName, ? extends Metric> metrics, TopicPartition tp) {
- try {
- final String resolvedTopic = tp.topic().replace('.', '_');
- final String resolvedPartition = String.valueOf(tp.partition());
- Predicate<Map.Entry<MetricName, ? extends Metric>> filter =
- entry -> {
- final MetricName metricName = entry.getKey();
- final Map<String, String> tags = metricName.tags();
-
- return metricName.group().equals(CONSUMER_FETCH_MANAGER_GROUP)
- && metricName.name().equals(RECORDS_LAG)
- && tags.containsKey("topic")
- && tags.get("topic").equals(resolvedTopic)
- && tags.containsKey("partition")
- && tags.get("partition").equals(resolvedPartition);
- };
- return MetricUtil.getKafkaMetric(metrics, filter);
- } catch (IllegalStateException e) {
- LOG.warn(
- String.format(
- "Error when getting Kafka consumer metric \"%s\" "
- + "for partition \"%s\". "
- + "Metric \"%s\" may not be reported correctly. ",
- RECORDS_LAG, tp, MetricNames.PENDING_RECORDS),
- e);
- return null;
- }
- }
-
- private static class Offset {
- long currentOffset;
- long committedOffset;
-
- Offset(long currentOffset, long committedOffset) {
- this.currentOffset = currentOffset;
- this.committedOffset = committedOffset;
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaPartitionSplitReader.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaPartitionSplitReader.java
deleted file mode 100644
index c440fc2edca..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaPartitionSplitReader.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.connector.source.SourceReaderContext;
-import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
-import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
-import org.apache.flink.connector.base.source.reader.splitreader.SplitsAddition;
-import org.apache.flink.connector.base.source.reader.splitreader.SplitsChange;
-import org.apache.flink.connector.kafka.source.KafkaSourceOptions;
-import org.apache.flink.connector.kafka.source.metrics.KafkaSourceReaderMetrics;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-import org.apache.flink.util.FlinkRuntimeException;
-import org.apache.flink.util.Preconditions;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.consumer.OffsetCommitCallback;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.WakeupException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringJoiner;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-/** A {@link SplitReader} implementation that reads records from Kafka partitions. */
-@Internal
-public class KafkaPartitionSplitReader
- implements SplitReader<ConsumerRecord<byte[], byte[]>, KafkaPartitionSplit> {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaPartitionSplitReader.class);
- private static final long POLL_TIMEOUT = 10000L;
-
- private final KafkaConsumer<byte[], byte[]> consumer;
- private final Map<TopicPartition, Long> stoppingOffsets;
- private final String groupId;
- private final int subtaskId;
-
- private final KafkaSourceReaderMetrics kafkaSourceReaderMetrics;
-
- // Tracking empty splits that has not been added to finished splits in fetch()
- private final Set<String> emptySplits = new HashSet<>();
-
- public KafkaPartitionSplitReader(
- Properties props,
- SourceReaderContext context,
- KafkaSourceReaderMetrics kafkaSourceReaderMetrics) {
- this.subtaskId = context.getIndexOfSubtask();
- this.kafkaSourceReaderMetrics = kafkaSourceReaderMetrics;
- Properties consumerProps = new Properties();
- consumerProps.putAll(props);
- consumerProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, createConsumerClientId(props));
- this.consumer = new KafkaConsumer<>(consumerProps);
- this.stoppingOffsets = new HashMap<>();
- this.groupId = consumerProps.getProperty(ConsumerConfig.GROUP_ID_CONFIG);
-
- // Metric registration
- maybeRegisterKafkaConsumerMetrics(props, kafkaSourceReaderMetrics, consumer);
- this.kafkaSourceReaderMetrics.registerNumBytesIn(consumer);
- }
-
- @Override
- public RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>> fetch() throws IOException {
- ConsumerRecords<byte[], byte[]> consumerRecords;
- try {
- consumerRecords = consumer.poll(Duration.ofMillis(POLL_TIMEOUT));
- } catch (WakeupException | IllegalStateException e) {
- // IllegalStateException will be thrown if the consumer is not assigned any partitions.
- // This happens if all assigned partitions are invalid or empty (starting offset >=
- // stopping offset). We just mark empty partitions as finished and return an empty
- // record container, and this consumer will be closed by SplitFetcherManager.
- KafkaPartitionSplitRecords recordsBySplits =
- new KafkaPartitionSplitRecords(
- ConsumerRecords.empty(), kafkaSourceReaderMetrics);
- markEmptySplitsAsFinished(recordsBySplits);
- return recordsBySplits;
- }
- KafkaPartitionSplitRecords recordsBySplits =
- new KafkaPartitionSplitRecords(consumerRecords, kafkaSourceReaderMetrics);
- List<TopicPartition> finishedPartitions = new ArrayList<>();
- for (TopicPartition tp : consumerRecords.partitions()) {
- long stoppingOffset = getStoppingOffset(tp);
- final List<ConsumerRecord<byte[], byte[]>> recordsFromPartition =
- consumerRecords.records(tp);
-
- if (recordsFromPartition.size() > 0) {
- final ConsumerRecord<byte[], byte[]> lastRecord =
- recordsFromPartition.get(recordsFromPartition.size() - 1);
-
- // After processing a record with offset of "stoppingOffset - 1", the split reader
- // should not continue fetching because the record with stoppingOffset may not
- // exist. Keep polling will just block forever.
- if (lastRecord.offset() >= stoppingOffset - 1) {
- recordsBySplits.setPartitionStoppingOffset(tp, stoppingOffset);
- finishSplitAtRecord(
- tp,
- stoppingOffset,
- lastRecord.offset(),
- finishedPartitions,
- recordsBySplits);
- }
- }
- // Track this partition's record lag if it never appears before
- kafkaSourceReaderMetrics.maybeAddRecordsLagMetric(consumer, tp);
- }
-
- markEmptySplitsAsFinished(recordsBySplits);
-
- // Unassign the partitions that has finished.
- if (!finishedPartitions.isEmpty()) {
- finishedPartitions.forEach(kafkaSourceReaderMetrics::removeRecordsLagMetric);
- unassignPartitions(finishedPartitions);
- }
-
- // Update numBytesIn
- kafkaSourceReaderMetrics.updateNumBytesInCounter();
-
- return recordsBySplits;
- }
-
- private void markEmptySplitsAsFinished(KafkaPartitionSplitRecords recordsBySplits) {
- // Some splits are discovered as empty when handling split additions. These splits should be
- // added to finished splits to clean up states in split fetcher and source reader.
- if (!emptySplits.isEmpty()) {
- recordsBySplits.finishedSplits.addAll(emptySplits);
- emptySplits.clear();
- }
- }
-
- @Override
- public void handleSplitsChanges(SplitsChange<KafkaPartitionSplit> splitsChange) {
- // Get all the partition assignments and stopping offsets.
- if (!(splitsChange instanceof SplitsAddition)) {
- throw new UnsupportedOperationException(
- String.format(
- "The SplitChange type of %s is not supported.",
- splitsChange.getClass()));
- }
-
- // Assignment.
- List<TopicPartition> newPartitionAssignments = new ArrayList<>();
- // Starting offsets.
- Map<TopicPartition, Long> partitionsStartingFromSpecifiedOffsets = new HashMap<>();
- List<TopicPartition> partitionsStartingFromEarliest = new ArrayList<>();
- List<TopicPartition> partitionsStartingFromLatest = new ArrayList<>();
- // Stopping offsets.
- List<TopicPartition> partitionsStoppingAtLatest = new ArrayList<>();
- Set<TopicPartition> partitionsStoppingAtCommitted = new HashSet<>();
-
- // Parse the starting and stopping offsets.
- splitsChange
- .splits()
- .forEach(
- s -> {
- newPartitionAssignments.add(s.getTopicPartition());
- parseStartingOffsets(
- s,
- partitionsStartingFromEarliest,
- partitionsStartingFromLatest,
- partitionsStartingFromSpecifiedOffsets);
- parseStoppingOffsets(
- s, partitionsStoppingAtLatest, partitionsStoppingAtCommitted);
- // Track the new topic partition in metrics
- kafkaSourceReaderMetrics.registerTopicPartition(s.getTopicPartition());
- });
-
- // Assign new partitions.
- newPartitionAssignments.addAll(consumer.assignment());
- consumer.assign(newPartitionAssignments);
-
- // Seek on the newly assigned partitions to their stating offsets.
- seekToStartingOffsets(
- partitionsStartingFromEarliest,
- partitionsStartingFromLatest,
- partitionsStartingFromSpecifiedOffsets);
- // Setup the stopping offsets.
- acquireAndSetStoppingOffsets(partitionsStoppingAtLatest, partitionsStoppingAtCommitted);
-
- // After acquiring the starting and stopping offsets, remove the empty splits if necessary.
- removeEmptySplits();
-
- maybeLogSplitChangesHandlingResult(splitsChange);
- }
-
- @Override
- public void wakeUp() {
- consumer.wakeup();
- }
-
- @Override
- public void close() throws Exception {
- consumer.close();
- }
-
- @Override
- public void pauseOrResumeSplits(
- Collection<KafkaPartitionSplit> splitsToPause,
- Collection<KafkaPartitionSplit> splitsToResume) {
- consumer.resume(
- splitsToResume.stream()
- .map(KafkaPartitionSplit::getTopicPartition)
- .collect(Collectors.toList()));
- consumer.pause(
- splitsToPause.stream()
- .map(KafkaPartitionSplit::getTopicPartition)
- .collect(Collectors.toList()));
- }
-
- // ---------------
-
- public void notifyCheckpointComplete(
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit,
- OffsetCommitCallback offsetCommitCallback) {
- consumer.commitAsync(offsetsToCommit, offsetCommitCallback);
- }
-
- @VisibleForTesting
- KafkaConsumer<byte[], byte[]> consumer() {
- return consumer;
- }
-
- // --------------- private helper method ----------------------
-
- private void parseStartingOffsets(
- KafkaPartitionSplit split,
- List<TopicPartition> partitionsStartingFromEarliest,
- List<TopicPartition> partitionsStartingFromLatest,
- Map<TopicPartition, Long> partitionsStartingFromSpecifiedOffsets) {
- TopicPartition tp = split.getTopicPartition();
- // Parse starting offsets.
- if (split.getStartingOffset() == KafkaPartitionSplit.EARLIEST_OFFSET) {
- partitionsStartingFromEarliest.add(tp);
- } else if (split.getStartingOffset() == KafkaPartitionSplit.LATEST_OFFSET) {
- partitionsStartingFromLatest.add(tp);
- } else if (split.getStartingOffset() == KafkaPartitionSplit.COMMITTED_OFFSET) {
- // Do nothing here, the consumer will first try to get the committed offsets of
- // these partitions by default.
- } else {
- partitionsStartingFromSpecifiedOffsets.put(tp, split.getStartingOffset());
- }
- }
-
- private void parseStoppingOffsets(
- KafkaPartitionSplit split,
- List<TopicPartition> partitionsStoppingAtLatest,
- Set<TopicPartition> partitionsStoppingAtCommitted) {
- TopicPartition tp = split.getTopicPartition();
- split.getStoppingOffset()
- .ifPresent(
- stoppingOffset -> {
- if (stoppingOffset >= 0) {
- stoppingOffsets.put(tp, stoppingOffset);
- } else if (stoppingOffset == KafkaPartitionSplit.LATEST_OFFSET) {
- partitionsStoppingAtLatest.add(tp);
- } else if (stoppingOffset == KafkaPartitionSplit.COMMITTED_OFFSET) {
- partitionsStoppingAtCommitted.add(tp);
- } else {
- // This should not happen.
- throw new FlinkRuntimeException(
- String.format(
- "Invalid stopping offset %d for partition %s",
- stoppingOffset, tp));
- }
- });
- }
-
- private void seekToStartingOffsets(
- List<TopicPartition> partitionsStartingFromEarliest,
- List<TopicPartition> partitionsStartingFromLatest,
- Map<TopicPartition, Long> partitionsStartingFromSpecifiedOffsets) {
-
- if (!partitionsStartingFromEarliest.isEmpty()) {
- LOG.trace("Seeking starting offsets to beginning: {}", partitionsStartingFromEarliest);
- consumer.seekToBeginning(partitionsStartingFromEarliest);
- }
-
- if (!partitionsStartingFromLatest.isEmpty()) {
- LOG.trace("Seeking starting offsets to end: {}", partitionsStartingFromLatest);
- consumer.seekToEnd(partitionsStartingFromLatest);
- }
-
- if (!partitionsStartingFromSpecifiedOffsets.isEmpty()) {
- LOG.trace(
- "Seeking starting offsets to specified offsets: {}",
- partitionsStartingFromSpecifiedOffsets);
- partitionsStartingFromSpecifiedOffsets.forEach(consumer::seek);
- }
- }
-
- private void acquireAndSetStoppingOffsets(
- List<TopicPartition> partitionsStoppingAtLatest,
- Set<TopicPartition> partitionsStoppingAtCommitted) {
- Map<TopicPartition, Long> endOffset = consumer.endOffsets(partitionsStoppingAtLatest);
- stoppingOffsets.putAll(endOffset);
- if (!partitionsStoppingAtCommitted.isEmpty()) {
- retryOnWakeup(
- () -> consumer.committed(partitionsStoppingAtCommitted),
- "getting committed offset as stopping offsets")
- .forEach(
- (tp, offsetAndMetadata) -> {
- Preconditions.checkNotNull(
- offsetAndMetadata,
- String.format(
- "Partition %s should stop at committed offset. "
- + "But there is no committed offset of this partition for group %s",
- tp, groupId));
- stoppingOffsets.put(tp, offsetAndMetadata.offset());
- });
- }
- }
-
- private void removeEmptySplits() {
- List<TopicPartition> emptyPartitions = new ArrayList<>();
- // If none of the partitions have any records,
- for (TopicPartition tp : consumer.assignment()) {
- if (retryOnWakeup(
- () -> consumer.position(tp),
- "getting starting offset to check if split is empty")
- >= getStoppingOffset(tp)) {
- emptyPartitions.add(tp);
- }
- }
- if (!emptyPartitions.isEmpty()) {
- LOG.debug(
- "These assigning splits are empty and will be marked as finished in later fetch: {}",
- emptyPartitions);
- // Add empty partitions to empty split set for later cleanup in fetch()
- emptySplits.addAll(
- emptyPartitions.stream()
- .map(KafkaPartitionSplit::toSplitId)
- .collect(Collectors.toSet()));
- // Un-assign partitions from Kafka consumer
- unassignPartitions(emptyPartitions);
- }
- }
-
- private void maybeLogSplitChangesHandlingResult(
- SplitsChange<KafkaPartitionSplit> splitsChange) {
- if (LOG.isDebugEnabled()) {
- StringJoiner splitsInfo = new StringJoiner(",");
- for (KafkaPartitionSplit split : splitsChange.splits()) {
- long startingOffset =
- retryOnWakeup(
- () -> consumer.position(split.getTopicPartition()),
- "logging starting position");
- long stoppingOffset = getStoppingOffset(split.getTopicPartition());
- splitsInfo.add(
- String.format(
- "[%s, start:%d, stop: %d]",
- split.getTopicPartition(), startingOffset, stoppingOffset));
- }
- LOG.debug("SplitsChange handling result: {}", splitsInfo);
- }
- }
-
- private void unassignPartitions(Collection<TopicPartition> partitionsToUnassign) {
- Collection<TopicPartition> newAssignment = new HashSet<>(consumer.assignment());
- newAssignment.removeAll(partitionsToUnassign);
- consumer.assign(newAssignment);
- }
-
- private String createConsumerClientId(Properties props) {
- String prefix = props.getProperty(KafkaSourceOptions.CLIENT_ID_PREFIX.key());
- return prefix + "-" + subtaskId;
- }
-
- private void finishSplitAtRecord(
- TopicPartition tp,
- long stoppingOffset,
- long currentOffset,
- List<TopicPartition> finishedPartitions,
- KafkaPartitionSplitRecords recordsBySplits) {
- LOG.debug(
- "{} has reached stopping offset {}, current offset is {}",
- tp,
- stoppingOffset,
- currentOffset);
- finishedPartitions.add(tp);
- recordsBySplits.addFinishedSplit(KafkaPartitionSplit.toSplitId(tp));
- }
-
- private long getStoppingOffset(TopicPartition tp) {
- return stoppingOffsets.getOrDefault(tp, Long.MAX_VALUE);
- }
-
- private void maybeRegisterKafkaConsumerMetrics(
- Properties props,
- KafkaSourceReaderMetrics kafkaSourceReaderMetrics,
- KafkaConsumer<?, ?> consumer) {
- final Boolean needToRegister =
- KafkaSourceOptions.getOption(
- props,
- KafkaSourceOptions.REGISTER_KAFKA_CONSUMER_METRICS,
- Boolean::parseBoolean);
- if (needToRegister) {
- kafkaSourceReaderMetrics.registerKafkaConsumerMetrics(consumer);
- }
- }
-
- /**
- * Catch {@link WakeupException} in Kafka consumer call and retry the invocation on exception.
- *
- * <p>This helper function handles a race condition as below:
- *
- * <ol>
- * <li>Fetcher thread finishes a {@link KafkaConsumer#poll(Duration)} call
- * <li>Task thread assigns new splits so invokes {@link #wakeUp()}, then the wakeup is
- * recorded and held by the consumer
- * <li>Later fetcher thread invokes {@link #handleSplitsChanges(SplitsChange)}, and
- * interactions with consumer will throw {@link WakeupException} because of the previously
- * held wakeup in the consumer
- * </ol>
- *
- * <p>Under this case we need to catch the {@link WakeupException} and retry the operation.
- */
- private <V> V retryOnWakeup(Supplier<V> consumerCall, String description) {
- try {
- return consumerCall.get();
- } catch (WakeupException we) {
- LOG.info(
- "Caught WakeupException while executing Kafka consumer call for {}. Will retry the consumer call.",
- description);
- return consumerCall.get();
- }
- }
-
- // ---------------- private helper class ------------------------
-
- private static class KafkaPartitionSplitRecords
- implements RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>> {
-
- private final Set<String> finishedSplits = new HashSet<>();
- private final Map<TopicPartition, Long> stoppingOffsets = new HashMap<>();
- private final ConsumerRecords<byte[], byte[]> consumerRecords;
- private final KafkaSourceReaderMetrics metrics;
- private final Iterator<TopicPartition> splitIterator;
- private Iterator<ConsumerRecord<byte[], byte[]>> recordIterator;
- private TopicPartition currentTopicPartition;
- private Long currentSplitStoppingOffset;
-
- private KafkaPartitionSplitRecords(
- ConsumerRecords<byte[], byte[]> consumerRecords, KafkaSourceReaderMetrics metrics) {
- this.consumerRecords = consumerRecords;
- this.splitIterator = consumerRecords.partitions().iterator();
- this.metrics = metrics;
- }
-
- private void setPartitionStoppingOffset(
- TopicPartition topicPartition, long stoppingOffset) {
- stoppingOffsets.put(topicPartition, stoppingOffset);
- }
-
- private void addFinishedSplit(String splitId) {
- finishedSplits.add(splitId);
- }
-
- @Nullable
- @Override
- public String nextSplit() {
- if (splitIterator.hasNext()) {
- currentTopicPartition = splitIterator.next();
- recordIterator = consumerRecords.records(currentTopicPartition).iterator();
- currentSplitStoppingOffset =
- stoppingOffsets.getOrDefault(currentTopicPartition, Long.MAX_VALUE);
- return currentTopicPartition.toString();
- } else {
- currentTopicPartition = null;
- recordIterator = null;
- currentSplitStoppingOffset = null;
- return null;
- }
- }
-
- @Nullable
- @Override
- public ConsumerRecord<byte[], byte[]> nextRecordFromSplit() {
- Preconditions.checkNotNull(
- currentTopicPartition,
- "Make sure nextSplit() did not return null before "
- + "iterate over the records split.");
- if (recordIterator.hasNext()) {
- final ConsumerRecord<byte[], byte[]> record = recordIterator.next();
- // Only emit records before stopping offset
- if (record.offset() < currentSplitStoppingOffset) {
- metrics.recordCurrentOffset(currentTopicPartition, record.offset());
- return record;
- }
- }
- return null;
- }
-
- @Override
- public Set<String> finishedSplits() {
- return finishedSplits;
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaRecordEmitter.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaRecordEmitter.java
deleted file mode 100644
index 3a1320a72dc..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaRecordEmitter.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.connector.source.SourceOutput;
-import org.apache.flink.connector.base.source.reader.RecordEmitter;
-import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitState;
-import org.apache.flink.util.Collector;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-
-import java.io.IOException;
-
-/** The {@link RecordEmitter} implementation for {@link KafkaSourceReader}. */
-@Internal
-public class KafkaRecordEmitter<T>
- implements RecordEmitter<ConsumerRecord<byte[], byte[]>, T, KafkaPartitionSplitState> {
-
- private final KafkaRecordDeserializationSchema<T> deserializationSchema;
- private final SourceOutputWrapper<T> sourceOutputWrapper = new SourceOutputWrapper<>();
-
- public KafkaRecordEmitter(KafkaRecordDeserializationSchema<T> deserializationSchema) {
- this.deserializationSchema = deserializationSchema;
- }
-
- @Override
- public void emitRecord(
- ConsumerRecord<byte[], byte[]> consumerRecord,
- SourceOutput<T> output,
- KafkaPartitionSplitState splitState)
- throws Exception {
- try {
- sourceOutputWrapper.setSourceOutput(output);
- sourceOutputWrapper.setTimestamp(consumerRecord.timestamp());
- deserializationSchema.deserialize(consumerRecord, sourceOutputWrapper);
- splitState.setCurrentOffset(consumerRecord.offset() + 1);
- } catch (Exception e) {
- throw new IOException("Failed to deserialize consumer record due to", e);
- }
- }
-
- private static class SourceOutputWrapper<T> implements Collector<T> {
-
- private SourceOutput<T> sourceOutput;
- private long timestamp;
-
- @Override
- public void collect(T record) {
- sourceOutput.collect(record, timestamp);
- }
-
- @Override
- public void close() {}
-
- private void setSourceOutput(SourceOutput<T> sourceOutput) {
- this.sourceOutput = sourceOutput;
- }
-
- private void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaSourceReader.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaSourceReader.java
deleted file mode 100644
index a3aa555f189..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/KafkaSourceReader.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.connector.source.SourceReaderContext;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.connector.base.source.reader.RecordEmitter;
-import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
-import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase;
-import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import org.apache.flink.connector.kafka.source.KafkaSourceOptions;
-import org.apache.flink.connector.kafka.source.metrics.KafkaSourceReaderMetrics;
-import org.apache.flink.connector.kafka.source.reader.fetcher.KafkaSourceFetcherManager;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplitState;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.common.TopicPartition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-/** The source reader for Kafka partitions. */
-@Internal
-public class KafkaSourceReader<T>
- extends SingleThreadMultiplexSourceReaderBase<
- ConsumerRecord<byte[], byte[]>, T, KafkaPartitionSplit, KafkaPartitionSplitState> {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceReader.class);
- // These maps need to be concurrent because it will be accessed by both the main thread
- // and the split fetcher thread in the callback.
- private final SortedMap<Long, Map<TopicPartition, OffsetAndMetadata>> offsetsToCommit;
- private final ConcurrentMap<TopicPartition, OffsetAndMetadata> offsetsOfFinishedSplits;
- private final KafkaSourceReaderMetrics kafkaSourceReaderMetrics;
- private final boolean commitOffsetsOnCheckpoint;
-
- public KafkaSourceReader(
- FutureCompletingBlockingQueue<RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>>>
- elementsQueue,
- KafkaSourceFetcherManager kafkaSourceFetcherManager,
- RecordEmitter<ConsumerRecord<byte[], byte[]>, T, KafkaPartitionSplitState>
- recordEmitter,
- Configuration config,
- SourceReaderContext context,
- KafkaSourceReaderMetrics kafkaSourceReaderMetrics) {
- super(elementsQueue, kafkaSourceFetcherManager, recordEmitter, config, context);
- this.offsetsToCommit = Collections.synchronizedSortedMap(new TreeMap<>());
- this.offsetsOfFinishedSplits = new ConcurrentHashMap<>();
- this.kafkaSourceReaderMetrics = kafkaSourceReaderMetrics;
- this.commitOffsetsOnCheckpoint =
- config.get(KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT);
- if (!commitOffsetsOnCheckpoint) {
- LOG.warn(
- "Offset commit on checkpoint is disabled. "
- + "Consuming offset will not be reported back to Kafka cluster.");
- }
- }
-
- @Override
- protected void onSplitFinished(Map<String, KafkaPartitionSplitState> finishedSplitIds) {
- finishedSplitIds.forEach(
- (ignored, splitState) -> {
- if (splitState.getCurrentOffset() >= 0) {
- offsetsOfFinishedSplits.put(
- splitState.getTopicPartition(),
- new OffsetAndMetadata(splitState.getCurrentOffset()));
- }
- });
- }
-
- @Override
- public List<KafkaPartitionSplit> snapshotState(long checkpointId) {
- List<KafkaPartitionSplit> splits = super.snapshotState(checkpointId);
- if (!commitOffsetsOnCheckpoint) {
- return splits;
- }
-
- if (splits.isEmpty() && offsetsOfFinishedSplits.isEmpty()) {
- offsetsToCommit.put(checkpointId, Collections.emptyMap());
- } else {
- Map<TopicPartition, OffsetAndMetadata> offsetsMap =
- offsetsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>());
- // Put the offsets of the active splits.
- for (KafkaPartitionSplit split : splits) {
- // If the checkpoint is triggered before the partition starting offsets
- // is retrieved, do not commit the offsets for those partitions.
- if (split.getStartingOffset() >= 0) {
- offsetsMap.put(
- split.getTopicPartition(),
- new OffsetAndMetadata(split.getStartingOffset()));
- }
- }
- // Put offsets of all the finished splits.
- offsetsMap.putAll(offsetsOfFinishedSplits);
- }
- return splits;
- }
-
- @Override
- public void notifyCheckpointComplete(long checkpointId) throws Exception {
- LOG.debug("Committing offsets for checkpoint {}", checkpointId);
- if (!commitOffsetsOnCheckpoint) {
- return;
- }
-
- Map<TopicPartition, OffsetAndMetadata> committedPartitions =
- offsetsToCommit.get(checkpointId);
- if (committedPartitions == null) {
- LOG.debug(
- "Offsets for checkpoint {} either do not exist or have already been committed.",
- checkpointId);
- return;
- }
-
- ((KafkaSourceFetcherManager) splitFetcherManager)
- .commitOffsets(
- committedPartitions,
- (ignored, e) -> {
- // The offset commit here is needed by the external monitoring. It won't
- // break Flink job's correctness if we fail to commit the offset here.
- if (e != null) {
- kafkaSourceReaderMetrics.recordFailedCommit();
- LOG.warn(
- "Failed to commit consumer offsets for checkpoint {}",
- checkpointId,
- e);
- } else {
- LOG.debug(
- "Successfully committed offsets for checkpoint {}",
- checkpointId);
- kafkaSourceReaderMetrics.recordSucceededCommit();
- // If the finished topic partition has been committed, we remove it
- // from the offsets of the finished splits map.
- committedPartitions.forEach(
- (tp, offset) ->
- kafkaSourceReaderMetrics.recordCommittedOffset(
- tp, offset.offset()));
- offsetsOfFinishedSplits
- .entrySet()
- .removeIf(
- entry ->
- committedPartitions.containsKey(
- entry.getKey()));
- while (!offsetsToCommit.isEmpty()
- && offsetsToCommit.firstKey() <= checkpointId) {
- offsetsToCommit.remove(offsetsToCommit.firstKey());
- }
- }
- });
- }
-
- @Override
- protected KafkaPartitionSplitState initializedState(KafkaPartitionSplit split) {
- return new KafkaPartitionSplitState(split);
- }
-
- @Override
- protected KafkaPartitionSplit toSplitType(String splitId, KafkaPartitionSplitState splitState) {
- return splitState.toKafkaPartitionSplit();
- }
-
- // ------------------------
-
- @VisibleForTesting
- SortedMap<Long, Map<TopicPartition, OffsetAndMetadata>> getOffsetsToCommit() {
- return offsetsToCommit;
- }
-
- @VisibleForTesting
- int getNumAliveFetchers() {
- return splitFetcherManager.getNumAliveFetchers();
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaDeserializationSchemaWrapper.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaDeserializationSchemaWrapper.java
deleted file mode 100644
index 94197e34714..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaDeserializationSchemaWrapper.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader.deserializer;
-
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema;
-import org.apache.flink.util.Collector;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-
-import java.io.IOException;
-
-/**
- * A wrapper class that wraps a {@link
- * org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema} to deserialize {@link
- * ConsumerRecord ConsumerRecords}.
- *
- * @param <T> the type of the deserialized records.
- */
-class KafkaDeserializationSchemaWrapper<T> implements KafkaRecordDeserializationSchema<T> {
- private static final long serialVersionUID = 1L;
- private final KafkaDeserializationSchema<T> kafkaDeserializationSchema;
-
- KafkaDeserializationSchemaWrapper(KafkaDeserializationSchema<T> kafkaDeserializationSchema) {
- this.kafkaDeserializationSchema = kafkaDeserializationSchema;
- }
-
- @Override
- public void open(DeserializationSchema.InitializationContext context) throws Exception {
- kafkaDeserializationSchema.open(context);
- }
-
- @Override
- public void deserialize(ConsumerRecord<byte[], byte[]> message, Collector<T> out)
- throws IOException {
- try {
- kafkaDeserializationSchema.deserialize(message, out);
- } catch (Exception exception) {
- throw new IOException(
- String.format("Failed to deserialize consumer record %s.", message), exception);
- }
- }
-
- @Override
- public TypeInformation<T> getProducedType() {
- return kafkaDeserializationSchema.getProducedType();
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaRecordDeserializationSchema.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaRecordDeserializationSchema.java
deleted file mode 100644
index 6ad6607c919..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaRecordDeserializationSchema.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader.deserializer;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema;
-import org.apache.flink.util.Collector;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.serialization.Deserializer;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.Map;
-
-/** An interface for the deserialization of Kafka records. */
-@PublicEvolving
-public interface KafkaRecordDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
-
- /**
- * Initialization method for the schema. It is called before the actual working methods {@link
- * #deserialize} and thus suitable for one time setup work.
- *
- * <p>The provided {@link DeserializationSchema.InitializationContext} can be used to access
- * additional features such as e.g. registering user metrics.
- *
- * @param context Contextual information that can be used during initialization.
- */
- default void open(DeserializationSchema.InitializationContext context) throws Exception {}
-
- /**
- * Deserializes the byte message.
- *
- * <p>Can output multiple records through the {@link Collector}. Note that number and size of
- * the produced records should be relatively small. Depending on the source implementation
- * records can be buffered in memory or collecting records might delay emitting checkpoint
- * barrier.
- *
- * @param record The ConsumerRecord to deserialize.
- * @param out The collector to put the resulting messages.
- */
- void deserialize(ConsumerRecord<byte[], byte[]> record, Collector<T> out) throws IOException;
-
- /**
- * Wraps a legacy {@link KafkaDeserializationSchema} as the deserializer of the {@link
- * ConsumerRecord ConsumerRecords}.
- *
- * <p>Note that the {@link KafkaDeserializationSchema#isEndOfStream(Object)} method will no
- * longer be used to determine the end of the stream.
- *
- * @param kafkaDeserializationSchema the legacy {@link KafkaDeserializationSchema} to use.
- * @param <V> the return type of the deserialized record.
- * @return A {@link KafkaRecordDeserializationSchema} that uses the given {@link
- * KafkaDeserializationSchema} to deserialize the {@link ConsumerRecord ConsumerRecords}.
- */
- static <V> KafkaRecordDeserializationSchema<V> of(
- KafkaDeserializationSchema<V> kafkaDeserializationSchema) {
- return new KafkaDeserializationSchemaWrapper<>(kafkaDeserializationSchema);
- }
-
- /**
- * Wraps a {@link DeserializationSchema} as the value deserialization schema of the {@link
- * ConsumerRecord ConsumerRecords}. The other fields such as key, headers, timestamp are
- * ignored.
- *
- * <p>Note that the {@link DeserializationSchema#isEndOfStream(Object)} method will no longer be
- * used to determine the end of the stream.
- *
- * @param valueDeserializationSchema the {@link DeserializationSchema} used to deserialized the
- * value of a {@link ConsumerRecord}.
- * @param <V> the type of the deserialized record.
- * @return A {@link KafkaRecordDeserializationSchema} that uses the given {@link
- * DeserializationSchema} to deserialize a {@link ConsumerRecord} from its value.
- */
- static <V> KafkaRecordDeserializationSchema<V> valueOnly(
- DeserializationSchema<V> valueDeserializationSchema) {
- return new KafkaValueOnlyDeserializationSchemaWrapper<>(valueDeserializationSchema);
- }
-
- /**
- * Wraps a Kafka {@link Deserializer} to a {@link KafkaRecordDeserializationSchema}.
- *
- * @param valueDeserializerClass the deserializer class used to deserialize the value.
- * @param <V> the value type.
- * @return A {@link KafkaRecordDeserializationSchema} that deserialize the value with the given
- * deserializer.
- */
- static <V> KafkaRecordDeserializationSchema<V> valueOnly(
- Class<? extends Deserializer<V>> valueDeserializerClass) {
- return valueOnly(valueDeserializerClass, Collections.emptyMap());
- }
-
- /**
- * Wraps a Kafka {@link Deserializer} to a {@link KafkaRecordDeserializationSchema}.
- *
- * @param valueDeserializerClass the deserializer class used to deserialize the value.
- * @param config the configuration of the value deserializer. If the deserializer is an
- * implementation of {@code Configurable}, the configuring logic will be handled by {@link
- * org.apache.kafka.common.Configurable#configure(Map)} with the given {@link config},
- * otherwise {@link Deserializer#configure(Map, boolean)} will be invoked.
- * @param <V> the value type.
- * @param <D> the type of the deserializer.
- * @return A {@link KafkaRecordDeserializationSchema} that deserialize the value with the given
- * deserializer.
- */
- static <V, D extends Deserializer<V>> KafkaRecordDeserializationSchema<V> valueOnly(
- Class<D> valueDeserializerClass, Map<String, String> config) {
- return new KafkaValueOnlyDeserializerWrapper<>(valueDeserializerClass, config);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializationSchemaWrapper.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializationSchemaWrapper.java
deleted file mode 100644
index 209f5e15c2c..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializationSchemaWrapper.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader.deserializer;
-
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.util.Collector;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-
-import java.io.IOException;
-
-/**
- * A class that wraps a {@link DeserializationSchema} as the value deserializer for a {@link
- * ConsumerRecord}.
- *
- * @param <T> the return type of the deserialization.
- */
-class KafkaValueOnlyDeserializationSchemaWrapper<T> implements KafkaRecordDeserializationSchema<T> {
- private static final long serialVersionUID = 1L;
- private final DeserializationSchema<T> deserializationSchema;
-
- KafkaValueOnlyDeserializationSchemaWrapper(DeserializationSchema<T> deserializationSchema) {
- this.deserializationSchema = deserializationSchema;
- }
-
- @Override
- public void open(DeserializationSchema.InitializationContext context) throws Exception {
- deserializationSchema.open(context);
- }
-
- @Override
- public void deserialize(ConsumerRecord<byte[], byte[]> message, Collector<T> out)
- throws IOException {
- deserializationSchema.deserialize(message.value(), out);
- }
-
- @Override
- public TypeInformation<T> getProducedType() {
- return deserializationSchema.getProducedType();
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializerWrapper.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializerWrapper.java
deleted file mode 100644
index 8c8095b6b0b..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/deserializer/KafkaValueOnlyDeserializerWrapper.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader.deserializer;
-
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.TemporaryClassLoaderContext;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.common.Configurable;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-/** A package private class to wrap {@link Deserializer}. */
-class KafkaValueOnlyDeserializerWrapper<T> implements KafkaRecordDeserializationSchema<T> {
-
- private static final long serialVersionUID = 5409547407386004054L;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(KafkaValueOnlyDeserializerWrapper.class);
-
- private final Class<? extends Deserializer<T>> deserializerClass;
-
- private final Map<String, String> config;
-
- private transient Deserializer<T> deserializer;
-
- KafkaValueOnlyDeserializerWrapper(
- Class<? extends Deserializer<T>> deserializerClass, Map<String, String> config) {
- this.deserializerClass = deserializerClass;
- this.config = config;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void open(DeserializationSchema.InitializationContext context) throws Exception {
- ClassLoader userCodeClassLoader = context.getUserCodeClassLoader().asClassLoader();
- try (TemporaryClassLoaderContext ignored =
- TemporaryClassLoaderContext.of(userCodeClassLoader)) {
- deserializer =
- (Deserializer<T>)
- InstantiationUtil.instantiate(
- deserializerClass.getName(),
- Deserializer.class,
- getClass().getClassLoader());
-
- if (deserializer instanceof Configurable) {
- ((Configurable) deserializer).configure(config);
- } else {
- // Always be false since this Deserializer is only used for value.
- deserializer.configure(config, false);
- }
- } catch (Exception e) {
- throw new IOException(
- "Failed to instantiate the deserializer of class " + deserializerClass, e);
- }
- }
-
- @Override
- public void deserialize(ConsumerRecord<byte[], byte[]> record, Collector<T> collector)
- throws IOException {
- if (deserializer == null) {
- throw new IllegalStateException(
- "The deserializer has not been created. Make sure the open() method has been "
- + "invoked.");
- }
-
- T value = deserializer.deserialize(record.topic(), record.value());
- LOG.trace(
- "Deserialized [partition: {}-{}, offset: {}, timestamp: {}, value: {}]",
- record.topic(),
- record.partition(),
- record.offset(),
- record.timestamp(),
- value);
- collector.collect(value);
- }
-
- @Override
- public TypeInformation<T> getProducedType() {
- return TypeExtractor.createTypeInfo(Deserializer.class, deserializerClass, 0, null, null);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/fetcher/KafkaSourceFetcherManager.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/fetcher/KafkaSourceFetcherManager.java
deleted file mode 100644
index 2113fb9c366..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/reader/fetcher/KafkaSourceFetcherManager.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.reader.fetcher;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
-import org.apache.flink.connector.base.source.reader.SourceReaderBase;
-import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager;
-import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher;
-import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherTask;
-import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
-import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import org.apache.flink.connector.kafka.source.reader.KafkaPartitionSplitReader;
-import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.consumer.OffsetCommitCallback;
-import org.apache.kafka.common.TopicPartition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-/**
- * The SplitFetcherManager for Kafka source. This class is needed to help commit the offsets to
- * Kafka using the KafkaConsumer inside the {@link
- * org.apache.flink.connector.kafka.source.reader.KafkaPartitionSplitReader}.
- */
-@Internal
-public class KafkaSourceFetcherManager
- extends SingleThreadFetcherManager<ConsumerRecord<byte[], byte[]>, KafkaPartitionSplit> {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceFetcherManager.class);
-
- /**
- * Creates a new SplitFetcherManager with a single I/O threads.
- *
- * @param elementsQueue The queue that is used to hand over data from the I/O thread (the
- * fetchers) to the reader (which emits the records and book-keeps the state. This must be
- * the same queue instance that is also passed to the {@link SourceReaderBase}.
- * @param splitReaderSupplier The factory for the split reader that connects to the source
- * system.
- * @param splitFinishedHook Hook for handling finished splits in split fetchers.
- */
- public KafkaSourceFetcherManager(
- FutureCompletingBlockingQueue<RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>>>
- elementsQueue,
- Supplier<SplitReader<ConsumerRecord<byte[], byte[]>, KafkaPartitionSplit>>
- splitReaderSupplier,
- Consumer<Collection<String>> splitFinishedHook) {
- super(elementsQueue, splitReaderSupplier, new Configuration(), splitFinishedHook);
- }
-
- public void commitOffsets(
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit, OffsetCommitCallback callback) {
- LOG.debug("Committing offsets {}", offsetsToCommit);
- if (offsetsToCommit.isEmpty()) {
- return;
- }
- SplitFetcher<ConsumerRecord<byte[], byte[]>, KafkaPartitionSplit> splitFetcher =
- fetchers.get(0);
- if (splitFetcher != null) {
- // The fetcher thread is still running. This should be the majority of the cases.
- enqueueOffsetsCommitTask(splitFetcher, offsetsToCommit, callback);
- } else {
- splitFetcher = createSplitFetcher();
- enqueueOffsetsCommitTask(splitFetcher, offsetsToCommit, callback);
- startFetcher(splitFetcher);
- }
- }
-
- private void enqueueOffsetsCommitTask(
- SplitFetcher<ConsumerRecord<byte[], byte[]>, KafkaPartitionSplit> splitFetcher,
- Map<TopicPartition, OffsetAndMetadata> offsetsToCommit,
- OffsetCommitCallback callback) {
- KafkaPartitionSplitReader kafkaReader =
- (KafkaPartitionSplitReader) splitFetcher.getSplitReader();
-
- splitFetcher.enqueueTask(
- new SplitFetcherTask() {
- @Override
- public boolean run() throws IOException {
- kafkaReader.notifyCheckpointComplete(offsetsToCommit, callback);
- return true;
- }
-
- @Override
- public void wakeUp() {}
- });
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplit.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplit.java
deleted file mode 100644
index 8c2a1fd1ffd..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplit.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.split;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.connector.source.SourceSplit;
-import org.apache.flink.util.FlinkRuntimeException;
-
-import org.apache.kafka.common.TopicPartition;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-
-/** A {@link SourceSplit} for a Kafka partition. */
-@Internal
-public class KafkaPartitionSplit implements SourceSplit {
- public static final long NO_STOPPING_OFFSET = Long.MIN_VALUE;
- // Indicating the split should consume from the latest.
- public static final long LATEST_OFFSET = -1;
- // Indicating the split should consume from the earliest.
- public static final long EARLIEST_OFFSET = -2;
- // Indicating the split should consume from the last committed offset.
- public static final long COMMITTED_OFFSET = -3;
-
- // Valid special starting offsets
- public static final Set<Long> VALID_STARTING_OFFSET_MARKERS =
- new HashSet<>(Arrays.asList(EARLIEST_OFFSET, LATEST_OFFSET, COMMITTED_OFFSET));
- public static final Set<Long> VALID_STOPPING_OFFSET_MARKERS =
- new HashSet<>(Arrays.asList(LATEST_OFFSET, COMMITTED_OFFSET, NO_STOPPING_OFFSET));
-
- private final TopicPartition tp;
- private final long startingOffset;
- private final long stoppingOffset;
-
- public KafkaPartitionSplit(TopicPartition tp, long startingOffset) {
- this(tp, startingOffset, NO_STOPPING_OFFSET);
- }
-
- public KafkaPartitionSplit(TopicPartition tp, long startingOffset, long stoppingOffset) {
- verifyInitialOffset(tp, startingOffset, stoppingOffset);
- this.tp = tp;
- this.startingOffset = startingOffset;
- this.stoppingOffset = stoppingOffset;
- }
-
- public String getTopic() {
- return tp.topic();
- }
-
- public int getPartition() {
- return tp.partition();
- }
-
- public TopicPartition getTopicPartition() {
- return tp;
- }
-
- public long getStartingOffset() {
- return startingOffset;
- }
-
- public Optional<Long> getStoppingOffset() {
- return stoppingOffset >= 0
- || stoppingOffset == LATEST_OFFSET
- || stoppingOffset == COMMITTED_OFFSET
- ? Optional.of(stoppingOffset)
- : Optional.empty();
- }
-
- @Override
- public String splitId() {
- return toSplitId(tp);
- }
-
- @Override
- public String toString() {
- return String.format(
- "[Partition: %s, StartingOffset: %d, StoppingOffset: %d]",
- tp, startingOffset, stoppingOffset);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(tp, startingOffset, stoppingOffset);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof KafkaPartitionSplit)) {
- return false;
- }
- KafkaPartitionSplit other = (KafkaPartitionSplit) obj;
- return tp.equals(other.tp)
- && startingOffset == other.startingOffset
- && stoppingOffset == other.stoppingOffset;
- }
-
- public static String toSplitId(TopicPartition tp) {
- return tp.toString();
- }
-
- // ------------ private methods ---------------
-
- private static void verifyInitialOffset(
- TopicPartition tp, Long startingOffset, long stoppingOffset) {
- if (startingOffset == null) {
- throw new FlinkRuntimeException(
- "Cannot initialize starting offset for partition " + tp);
- }
-
- if (startingOffset < 0 && !VALID_STARTING_OFFSET_MARKERS.contains(startingOffset)) {
- throw new FlinkRuntimeException(
- String.format(
- "Invalid starting offset %d is specified for partition %s. "
- + "It should either be non-negative or be one of the "
- + "[%d(earliest), %d(latest), %d(committed)].",
- startingOffset, tp, LATEST_OFFSET, EARLIEST_OFFSET, COMMITTED_OFFSET));
- }
-
- if (stoppingOffset < 0 && !VALID_STOPPING_OFFSET_MARKERS.contains(stoppingOffset)) {
- throw new FlinkRuntimeException(
- String.format(
- "Illegal stopping offset %d is specified for partition %s. "
- + "It should either be non-negative or be one of the "
- + "[%d(latest), %d(committed), %d(Long.MIN_VALUE, no_stopping_offset)].",
- stoppingOffset,
- tp,
- LATEST_OFFSET,
- COMMITTED_OFFSET,
- NO_STOPPING_OFFSET));
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitSerializer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitSerializer.java
deleted file mode 100644
index d906a0bc522..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitSerializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.split;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
-
-import org.apache.kafka.common.TopicPartition;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * The {@link org.apache.flink.core.io.SimpleVersionedSerializer serializer} for {@link
- * KafkaPartitionSplit}.
- */
-@Internal
-public class KafkaPartitionSplitSerializer
- implements SimpleVersionedSerializer<KafkaPartitionSplit> {
-
- private static final int CURRENT_VERSION = 0;
-
- @Override
- public int getVersion() {
- return CURRENT_VERSION;
- }
-
- @Override
- public byte[] serialize(KafkaPartitionSplit split) throws IOException {
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos)) {
- out.writeUTF(split.getTopic());
- out.writeInt(split.getPartition());
- out.writeLong(split.getStartingOffset());
- out.writeLong(split.getStoppingOffset().orElse(KafkaPartitionSplit.NO_STOPPING_OFFSET));
- out.flush();
- return baos.toByteArray();
- }
- }
-
- @Override
- public KafkaPartitionSplit deserialize(int version, byte[] serialized) throws IOException {
- try (ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
- DataInputStream in = new DataInputStream(bais)) {
- String topic = in.readUTF();
- int partition = in.readInt();
- long offset = in.readLong();
- long stoppingOffset = in.readLong();
- return new KafkaPartitionSplit(
- new TopicPartition(topic, partition), offset, stoppingOffset);
- }
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitState.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitState.java
deleted file mode 100644
index 18a9e656d6b..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/connector/kafka/source/split/KafkaPartitionSplitState.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.connector.kafka.source.split;
-
-import org.apache.flink.annotation.Internal;
-
-/** This class extends KafkaPartitionSplit to track a mutable current offset. */
-@Internal
-public class KafkaPartitionSplitState extends KafkaPartitionSplit {
-
- private long currentOffset;
-
- public KafkaPartitionSplitState(KafkaPartitionSplit partitionSplit) {
- super(
- partitionSplit.getTopicPartition(),
- partitionSplit.getStartingOffset(),
- partitionSplit.getStoppingOffset().orElse(NO_STOPPING_OFFSET));
- this.currentOffset = partitionSplit.getStartingOffset();
- }
-
- public long getCurrentOffset() {
- return currentOffset;
- }
-
- public void setCurrentOffset(long currentOffset) {
- this.currentOffset = currentOffset;
- }
-
- /**
- * Use the current offset as the starting offset to create a new KafkaPartitionSplit.
- *
- * @return a new KafkaPartitionSplit which uses the current offset as its starting offset.
- */
- public KafkaPartitionSplit toKafkaPartitionSplit() {
- return new KafkaPartitionSplit(
- getTopicPartition(),
- getCurrentOffset(),
- getStoppingOffset().orElse(NO_STOPPING_OFFSET));
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer.java
deleted file mode 100644
index 1e506a8f205..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
-import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode;
-import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher;
-import org.apache.flink.streaming.connectors.kafka.internals.AbstractPartitionDiscoverer;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaDeserializationSchemaWrapper;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaFetcher;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaPartitionDiscoverer;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor;
-import org.apache.flink.util.PropertiesUtil;
-import org.apache.flink.util.SerializedValue;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.serialization.ByteArrayDeserializer;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.regex.Pattern;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.PropertiesUtil.getBoolean;
-import static org.apache.flink.util.PropertiesUtil.getLong;
-
-/**
- * The Flink Kafka Consumer is a streaming data source that pulls a parallel data stream from Apache
- * Kafka. The consumer can run in multiple parallel instances, each of which will pull data from one
- * or more Kafka partitions.
- *
- * <p>The Flink Kafka Consumer participates in checkpointing and guarantees that no data is lost
- * during a failure, and that the computation processes elements "exactly once". (Note: These
- * guarantees naturally assume that Kafka itself does not lose any data.)
- *
- * <p>Please note that Flink snapshots the offsets internally as part of its distributed
- * checkpoints. The offsets committed to Kafka are only to bring the outside view of progress in
- * sync with Flink's view of the progress. That way, monitoring and other jobs can get a view of how
- * far the Flink Kafka consumer has consumed a topic.
- *
- * <p>Please refer to Kafka's documentation for the available configuration properties:
- * http://kafka.apache.org/documentation.html#newconsumerconfigs
- */
-@PublicEvolving
-@Deprecated
-public class FlinkKafkaConsumer<T> extends FlinkKafkaConsumerBase<T> {
-
- private static final long serialVersionUID = 1L;
-
- /** Configuration key to change the polling timeout. * */
- public static final String KEY_POLL_TIMEOUT = "flink.poll-timeout";
-
- /**
- * From Kafka's Javadoc: The time, in milliseconds, spent waiting in poll if data is not
- * available. If 0, returns immediately with any records that are available now.
- */
- public static final long DEFAULT_POLL_TIMEOUT = 100L;
-
- // ------------------------------------------------------------------------
-
- /** User-supplied properties for Kafka. * */
- protected final Properties properties;
-
- /**
- * From Kafka's Javadoc: The time, in milliseconds, spent waiting in poll if data is not
- * available. If 0, returns immediately with any records that are available now
- */
- protected final long pollTimeout;
-
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new Kafka streaming source consumer.
- *
- * @param topic The name of the topic that should be consumed.
- * @param valueDeserializer The de-/serializer used to convert between Kafka's byte messages and
- * Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- String topic, DeserializationSchema<T> valueDeserializer, Properties props) {
- this(Collections.singletonList(topic), valueDeserializer, props);
- }
-
- /**
- * Creates a new Kafka streaming source consumer.
- *
- * <p>This constructor allows passing a {@see KafkaDeserializationSchema} for reading key/value
- * pairs, offsets, and topic names from Kafka.
- *
- * @param topic The name of the topic that should be consumed.
- * @param deserializer The keyed de-/serializer used to convert between Kafka's byte messages
- * and Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- String topic, KafkaDeserializationSchema<T> deserializer, Properties props) {
- this(Collections.singletonList(topic), deserializer, props);
- }
-
- /**
- * Creates a new Kafka streaming source consumer.
- *
- * <p>This constructor allows passing multiple topics to the consumer.
- *
- * @param topics The Kafka topics to read from.
- * @param deserializer The de-/serializer used to convert between Kafka's byte messages and
- * Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- List<String> topics, DeserializationSchema<T> deserializer, Properties props) {
- this(topics, new KafkaDeserializationSchemaWrapper<>(deserializer), props);
- }
-
- /**
- * Creates a new Kafka streaming source consumer.
- *
- * <p>This constructor allows passing multiple topics and a key/value deserialization schema.
- *
- * @param topics The Kafka topics to read from.
- * @param deserializer The keyed de-/serializer used to convert between Kafka's byte messages
- * and Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- List<String> topics, KafkaDeserializationSchema<T> deserializer, Properties props) {
- this(topics, null, deserializer, props);
- }
-
- /**
- * Creates a new Kafka streaming source consumer. Use this constructor to subscribe to multiple
- * topics based on a regular expression pattern.
- *
- * <p>If partition discovery is enabled (by setting a non-negative value for {@link
- * FlinkKafkaConsumer#KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS} in the properties), topics with
- * names matching the pattern will also be subscribed to as they are created on the fly.
- *
- * @param subscriptionPattern The regular expression for a pattern of topic names to subscribe
- * to.
- * @param valueDeserializer The de-/serializer used to convert between Kafka's byte messages and
- * Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- Pattern subscriptionPattern,
- DeserializationSchema<T> valueDeserializer,
- Properties props) {
- this(
- null,
- subscriptionPattern,
- new KafkaDeserializationSchemaWrapper<>(valueDeserializer),
- props);
- }
-
- /**
- * Creates a new Kafka streaming source consumer. Use this constructor to subscribe to multiple
- * topics based on a regular expression pattern.
- *
- * <p>If partition discovery is enabled (by setting a non-negative value for {@link
- * FlinkKafkaConsumer#KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS} in the properties), topics with
- * names matching the pattern will also be subscribed to as they are created on the fly.
- *
- * <p>This constructor allows passing a {@see KafkaDeserializationSchema} for reading key/value
- * pairs, offsets, and topic names from Kafka.
- *
- * @param subscriptionPattern The regular expression for a pattern of topic names to subscribe
- * to.
- * @param deserializer The keyed de-/serializer used to convert between Kafka's byte messages
- * and Flink's objects.
- * @param props
- */
- public FlinkKafkaConsumer(
- Pattern subscriptionPattern,
- KafkaDeserializationSchema<T> deserializer,
- Properties props) {
- this(null, subscriptionPattern, deserializer, props);
- }
-
- private FlinkKafkaConsumer(
- List<String> topics,
- Pattern subscriptionPattern,
- KafkaDeserializationSchema<T> deserializer,
- Properties props) {
-
- super(
- topics,
- subscriptionPattern,
- deserializer,
- getLong(
- checkNotNull(props, "props"),
- KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS,
- PARTITION_DISCOVERY_DISABLED),
- !getBoolean(props, KEY_DISABLE_METRICS, false));
-
- this.properties = props;
- setDeserializer(this.properties);
-
- // configure the polling timeout
- try {
- if (properties.containsKey(KEY_POLL_TIMEOUT)) {
- this.pollTimeout = Long.parseLong(properties.getProperty(KEY_POLL_TIMEOUT));
- } else {
- this.pollTimeout = DEFAULT_POLL_TIMEOUT;
- }
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Cannot parse poll timeout for '" + KEY_POLL_TIMEOUT + '\'', e);
- }
- }
-
- @Override
- protected AbstractFetcher<T, ?> createFetcher(
- SourceContext<T> sourceContext,
- Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
- SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
- StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode,
- MetricGroup consumerMetricGroup,
- boolean useMetrics)
- throws Exception {
-
- // make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
- // this overwrites whatever setting the user configured in the properties
- adjustAutoCommitConfig(properties, offsetCommitMode);
-
- return new KafkaFetcher<>(
- sourceContext,
- assignedPartitionsWithInitialOffsets,
- watermarkStrategy,
- runtimeContext.getProcessingTimeService(),
- runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
- runtimeContext.getUserCodeClassLoader(),
- runtimeContext.getTaskNameWithSubtasks(),
- deserializer,
- properties,
- pollTimeout,
- runtimeContext.getMetricGroup(),
- consumerMetricGroup,
- useMetrics);
- }
-
- @Override
- protected AbstractPartitionDiscoverer createPartitionDiscoverer(
- KafkaTopicsDescriptor topicsDescriptor,
- int indexOfThisSubtask,
- int numParallelSubtasks) {
-
- return new KafkaPartitionDiscoverer(
- topicsDescriptor, indexOfThisSubtask, numParallelSubtasks, properties);
- }
-
- @Override
- protected Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp(
- Collection<KafkaTopicPartition> partitions, long timestamp) {
-
- Map<TopicPartition, Long> partitionOffsetsRequest = new HashMap<>(partitions.size());
- for (KafkaTopicPartition partition : partitions) {
- partitionOffsetsRequest.put(
- new TopicPartition(partition.getTopic(), partition.getPartition()), timestamp);
- }
-
- final Map<KafkaTopicPartition, Long> result = new HashMap<>(partitions.size());
- // use a short-lived consumer to fetch the offsets;
- // this is ok because this is a one-time operation that happens only on startup
- try (KafkaConsumer<?, ?> consumer = new KafkaConsumer(properties)) {
- for (Map.Entry<TopicPartition, OffsetAndTimestamp> partitionToOffset :
- consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) {
-
- result.put(
- new KafkaTopicPartition(
- partitionToOffset.getKey().topic(),
- partitionToOffset.getKey().partition()),
- (partitionToOffset.getValue() == null)
- ? null
- : partitionToOffset.getValue().offset());
- }
- }
- return result;
- }
-
- @Override
- protected boolean getIsAutoCommitEnabled() {
- return getBoolean(properties, ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true)
- && PropertiesUtil.getLong(
- properties, ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 5000)
- > 0;
- }
-
- /**
- * Makes sure that the ByteArrayDeserializer is registered in the Kafka properties.
- *
- * @param props The Kafka properties to register the serializer in.
- */
- private static void setDeserializer(Properties props) {
- final String deSerName = ByteArrayDeserializer.class.getName();
-
- Object keyDeSer = props.get(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG);
- Object valDeSer = props.get(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG);
-
- if (keyDeSer != null && !keyDeSer.equals(deSerName)) {
- LOG.warn(
- "Ignoring configured key DeSerializer ({})",
- ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG);
- }
- if (valDeSer != null && !valDeSer.equals(deSerName)) {
- LOG.warn(
- "Ignoring configured value DeSerializer ({})",
- ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG);
- }
-
- props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, deSerName);
- props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, deSerName);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
deleted file mode 100644
index f291b05bce9..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++ /dev/null
@@ -1,1227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.api.common.serialization.RuntimeContextInitializationContextAdapters;
-import org.apache.flink.api.common.state.CheckpointListener;
-import org.apache.flink.api.common.state.ListState;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.state.OperatorStateStore;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.base.LongSerializer;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.runtime.TupleSerializer;
-import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.runtime.state.FunctionInitializationContext;
-import org.apache.flink.runtime.state.FunctionSnapshotContext;
-import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
-import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
-import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode;
-import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitModes;
-import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
-import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher;
-import org.apache.flink.streaming.connectors.kafka.internals.AbstractPartitionDiscoverer;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionAssigner;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionStateSentinel;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor;
-import org.apache.flink.streaming.runtime.operators.util.AssignerWithPeriodicWatermarksAdapter;
-import org.apache.flink.streaming.runtime.operators.util.AssignerWithPunctuatedWatermarksAdapter;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.SerializedValue;
-
-import org.apache.commons.collections.map.LinkedMap;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Pattern;
-
-import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.COMMITS_FAILED_METRICS_COUNTER;
-import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.COMMITS_SUCCEEDED_METRICS_COUNTER;
-import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.KAFKA_CONSUMER_METRICS_GROUP;
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * Base class of all Flink Kafka Consumer data sources. This implements the common behavior across
- * all Kafka versions.
- *
- * <p>The Kafka version specific behavior is defined mainly in the specific subclasses of the {@link
- * AbstractFetcher}.
- *
- * @param <T> The type of records produced by this data source
- */
-@Internal
-public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFunction<T>
- implements CheckpointListener, ResultTypeQueryable<T>, CheckpointedFunction {
-
- private static final long serialVersionUID = -6272159445203409112L;
-
- protected static final Logger LOG = LoggerFactory.getLogger(FlinkKafkaConsumerBase.class);
-
- /** The maximum number of pending non-committed checkpoints to track, to avoid memory leaks. */
- public static final int MAX_NUM_PENDING_CHECKPOINTS = 100;
-
- /**
- * The default interval to execute partition discovery, in milliseconds ({@code Long.MIN_VALUE},
- * i.e. disabled by default).
- */
- public static final long PARTITION_DISCOVERY_DISABLED = Long.MIN_VALUE;
-
- /** Boolean configuration key to disable metrics tracking. * */
- public static final String KEY_DISABLE_METRICS = "flink.disable-metrics";
-
- /** Configuration key to define the consumer's partition discovery interval, in milliseconds. */
- public static final String KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS =
- "flink.partition-discovery.interval-millis";
-
- /** State name of the consumer's partition offset states. */
- private static final String OFFSETS_STATE_NAME = "topic-partition-offset-states";
-
- // ------------------------------------------------------------------------
- // configuration state, set on the client relevant for all subtasks
- // ------------------------------------------------------------------------
-
- /** Describes whether we are discovering partitions for fixed topics or a topic pattern. */
- private final KafkaTopicsDescriptor topicsDescriptor;
-
- /** The schema to convert between Kafka's byte messages, and Flink's objects. */
- protected final KafkaDeserializationSchema<T> deserializer;
-
- /**
- * The set of topic partitions that the source will read, with their initial offsets to start
- * reading from.
- */
- private Map<KafkaTopicPartition, Long> subscribedPartitionsToStartOffsets;
-
- /**
- * Optional watermark strategy that will be run per Kafka partition, to exploit per-partition
- * timestamp characteristics. The watermark strategy is kept in serialized form, to deserialize
- * it into multiple copies.
- */
- private SerializedValue<WatermarkStrategy<T>> watermarkStrategy;
-
- /**
- * User-set flag determining whether or not to commit on checkpoints. Note: this flag does not
- * represent the final offset commit mode.
- */
- private boolean enableCommitOnCheckpoints = true;
-
- /** User-set flag to disable filtering restored partitions with current topics descriptor. */
- private boolean filterRestoredPartitionsWithCurrentTopicsDescriptor = true;
-
- /**
- * The offset commit mode for the consumer. The value of this can only be determined in {@link
- * FlinkKafkaConsumerBase#open(Configuration)} since it depends on whether or not checkpointing
- * is enabled for the job.
- */
- private OffsetCommitMode offsetCommitMode;
-
- /** User configured value for discovery interval, in milliseconds. */
- private final long discoveryIntervalMillis;
-
- /** The startup mode for the consumer (default is {@link StartupMode#GROUP_OFFSETS}). */
- private StartupMode startupMode = StartupMode.GROUP_OFFSETS;
-
- /**
- * Specific startup offsets; only relevant when startup mode is {@link
- * StartupMode#SPECIFIC_OFFSETS}.
- */
- private Map<KafkaTopicPartition, Long> specificStartupOffsets;
-
- /**
- * Timestamp to determine startup offsets; only relevant when startup mode is {@link
- * StartupMode#TIMESTAMP}.
- */
- private Long startupOffsetsTimestamp;
-
- // ------------------------------------------------------------------------
- // runtime state (used individually by each parallel subtask)
- // ------------------------------------------------------------------------
-
- /** Data for pending but uncommitted offsets. */
- private final LinkedMap pendingOffsetsToCommit = new LinkedMap();
-
- /** The fetcher implements the connections to the Kafka brokers. */
- private transient volatile AbstractFetcher<T, ?> kafkaFetcher;
-
- /** The partition discoverer, used to find new partitions. */
- private transient volatile AbstractPartitionDiscoverer partitionDiscoverer;
-
- /**
- * The offsets to restore to, if the consumer restores state from a checkpoint.
- *
- * <p>This map will be populated by the {@link #initializeState(FunctionInitializationContext)}
- * method.
- *
- * <p>Using a sorted map as the ordering is important when using restored state to seed the
- * partition discoverer.
- */
- private transient volatile TreeMap<KafkaTopicPartition, Long> restoredState;
-
- /** Accessor for state in the operator state backend. */
- private transient ListState<Tuple2<KafkaTopicPartition, Long>> unionOffsetStates;
-
- /** Discovery loop, executed in a separate thread. */
- private transient volatile Thread discoveryLoopThread;
-
- /** Flag indicating whether the consumer is still running. */
- private volatile boolean running = true;
-
- // ------------------------------------------------------------------------
- // internal metrics
- // ------------------------------------------------------------------------
-
- /**
- * Flag indicating whether or not metrics should be exposed. If {@code true}, offset metrics
- * (e.g. current offset, committed offset) and Kafka-shipped metrics will be registered.
- */
- private final boolean useMetrics;
-
- /** Counter for successful Kafka offset commits. */
- private transient Counter successfulCommits;
-
- /** Counter for failed Kafka offset commits. */
- private transient Counter failedCommits;
-
- /**
- * Callback interface that will be invoked upon async Kafka commit completion. Please be aware
- * that default callback implementation in base class does not provide any guarantees on
- * thread-safety. This is sufficient for now because current supported Kafka connectors
- * guarantee no more than 1 concurrent async pending offset commit.
- */
- private transient KafkaCommitCallback offsetCommitCallback;
-
- // ------------------------------------------------------------------------
-
- /**
- * Base constructor.
- *
- * @param topics fixed list of topics to subscribe to (null, if using topic pattern)
- * @param topicPattern the topic pattern to subscribe to (null, if using fixed topics)
- * @param deserializer The deserializer to turn raw byte messages into Java/Scala objects.
- * @param discoveryIntervalMillis the topic / partition discovery interval, in milliseconds (0
- * if discovery is disabled).
- */
- public FlinkKafkaConsumerBase(
- List<String> topics,
- Pattern topicPattern,
- KafkaDeserializationSchema<T> deserializer,
- long discoveryIntervalMillis,
- boolean useMetrics) {
- this.topicsDescriptor = new KafkaTopicsDescriptor(topics, topicPattern);
- this.deserializer = checkNotNull(deserializer, "valueDeserializer");
-
- checkArgument(
- discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED
- || discoveryIntervalMillis >= 0,
- "Cannot define a negative value for the topic / partition discovery interval.");
- this.discoveryIntervalMillis = discoveryIntervalMillis;
-
- this.useMetrics = useMetrics;
- }
-
- /**
- * Make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS. This
- * overwrites whatever setting the user configured in the properties.
- *
- * @param properties - Kafka configuration properties to be adjusted
- * @param offsetCommitMode offset commit mode
- */
- protected static void adjustAutoCommitConfig(
- Properties properties, OffsetCommitMode offsetCommitMode) {
- if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS
- || offsetCommitMode == OffsetCommitMode.DISABLED) {
- properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
- }
- }
- // ------------------------------------------------------------------------
- // Configuration
- // ------------------------------------------------------------------------
-
- /**
- * Sets the given {@link WatermarkStrategy} on this consumer. These will be used to assign
- * timestamps to records and generates watermarks to signal event time progress.
- *
- * <p>Running timestamp extractors / watermark generators directly inside the Kafka source
- * (which you can do by using this method), per Kafka partition, allows users to let them
- * exploit the per-partition characteristics.
- *
- * <p>When a subtask of a FlinkKafkaConsumer source reads multiple Kafka partitions, the streams
- * from the partitions are unioned in a "first come first serve" fashion. Per-partition
- * characteristics are usually lost that way. For example, if the timestamps are strictly
- * ascending per Kafka partition, they will not be strictly ascending in the resulting Flink
- * DataStream, if the parallel source subtask reads more than one partition.
- *
- * <p>Common watermark generation patterns can be found as static methods in the {@link
- * org.apache.flink.api.common.eventtime.WatermarkStrategy} class.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> assignTimestampsAndWatermarks(
- WatermarkStrategy<T> watermarkStrategy) {
- checkNotNull(watermarkStrategy);
-
- try {
- ClosureCleaner.clean(
- watermarkStrategy, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
- this.watermarkStrategy = new SerializedValue<>(watermarkStrategy);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "The given WatermarkStrategy is not serializable", e);
- }
-
- return this;
- }
-
- /**
- * Specifies an {@link AssignerWithPunctuatedWatermarks} to emit watermarks in a punctuated
- * manner. The watermark extractor will run per Kafka partition, watermarks will be merged
- * across partitions in the same way as in the Flink runtime, when streams are merged.
- *
- * <p>When a subtask of a FlinkKafkaConsumer source reads multiple Kafka partitions, the streams
- * from the partitions are unioned in a "first come first serve" fashion. Per-partition
- * characteristics are usually lost that way. For example, if the timestamps are strictly
- * ascending per Kafka partition, they will not be strictly ascending in the resulting Flink
- * DataStream, if the parallel source subtask reads more than one partition.
- *
- * <p>Running timestamp extractors / watermark generators directly inside the Kafka source, per
- * Kafka partition, allows users to let them exploit the per-partition characteristics.
- *
- * <p>Note: One can use either an {@link AssignerWithPunctuatedWatermarks} or an {@link
- * AssignerWithPeriodicWatermarks}, not both at the same time.
- *
- * <p>This method uses the deprecated watermark generator interfaces. Please switch to {@link
- * #assignTimestampsAndWatermarks(WatermarkStrategy)} to use the new interfaces instead. The new
- * interfaces support watermark idleness and no longer need to differentiate between "periodic"
- * and "punctuated" watermarks.
- *
- * @deprecated Please use {@link #assignTimestampsAndWatermarks(WatermarkStrategy)} instead.
- * @param assigner The timestamp assigner / watermark generator to use.
- * @return The consumer object, to allow function chaining.
- */
- @Deprecated
- public FlinkKafkaConsumerBase<T> assignTimestampsAndWatermarks(
- AssignerWithPunctuatedWatermarks<T> assigner) {
- checkNotNull(assigner);
-
- if (this.watermarkStrategy != null) {
- throw new IllegalStateException("Some watermark strategy has already been set.");
- }
-
- try {
- ClosureCleaner.clean(assigner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
- final WatermarkStrategy<T> wms =
- new AssignerWithPunctuatedWatermarksAdapter.Strategy<>(assigner);
-
- return assignTimestampsAndWatermarks(wms);
- } catch (Exception e) {
- throw new IllegalArgumentException("The given assigner is not serializable", e);
- }
- }
-
- /**
- * Specifies an {@link AssignerWithPunctuatedWatermarks} to emit watermarks in a punctuated
- * manner. The watermark extractor will run per Kafka partition, watermarks will be merged
- * across partitions in the same way as in the Flink runtime, when streams are merged.
- *
- * <p>When a subtask of a FlinkKafkaConsumer source reads multiple Kafka partitions, the streams
- * from the partitions are unioned in a "first come first serve" fashion. Per-partition
- * characteristics are usually lost that way. For example, if the timestamps are strictly
- * ascending per Kafka partition, they will not be strictly ascending in the resulting Flink
- * DataStream, if the parallel source subtask reads more that one partition.
- *
- * <p>Running timestamp extractors / watermark generators directly inside the Kafka source, per
- * Kafka partition, allows users to let them exploit the per-partition characteristics.
- *
- * <p>Note: One can use either an {@link AssignerWithPunctuatedWatermarks} or an {@link
- * AssignerWithPeriodicWatermarks}, not both at the same time.
- *
- * <p>This method uses the deprecated watermark generator interfaces. Please switch to {@link
- * #assignTimestampsAndWatermarks(WatermarkStrategy)} to use the new interfaces instead. The new
- * interfaces support watermark idleness and no longer need to differentiate between "periodic"
- * and "punctuated" watermarks.
- *
- * @deprecated Please use {@link #assignTimestampsAndWatermarks(WatermarkStrategy)} instead.
- * @param assigner The timestamp assigner / watermark generator to use.
- * @return The consumer object, to allow function chaining.
- */
- @Deprecated
- public FlinkKafkaConsumerBase<T> assignTimestampsAndWatermarks(
- AssignerWithPeriodicWatermarks<T> assigner) {
- checkNotNull(assigner);
-
- if (this.watermarkStrategy != null) {
- throw new IllegalStateException("Some watermark strategy has already been set.");
- }
-
- try {
- ClosureCleaner.clean(assigner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
- final WatermarkStrategy<T> wms =
- new AssignerWithPeriodicWatermarksAdapter.Strategy<>(assigner);
-
- return assignTimestampsAndWatermarks(wms);
- } catch (Exception e) {
- throw new IllegalArgumentException("The given assigner is not serializable", e);
- }
- }
-
- /**
- * Specifies whether or not the consumer should commit offsets back to Kafka on checkpoints.
- *
- * <p>This setting will only have effect if checkpointing is enabled for the job. If
- * checkpointing isn't enabled, only the "auto.commit.enable" (for 0.8) / "enable.auto.commit"
- * (for 0.9+) property settings will be used.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setCommitOffsetsOnCheckpoints(boolean commitOnCheckpoints) {
- this.enableCommitOnCheckpoints = commitOnCheckpoints;
- return this;
- }
-
- /**
- * Specifies the consumer to start reading from the earliest offset for all partitions. This
- * lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
- *
- * <p>This method does not affect where partitions are read from when the consumer is restored
- * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or savepoint,
- * only the offsets in the restored state will be used.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setStartFromEarliest() {
- this.startupMode = StartupMode.EARLIEST;
- this.startupOffsetsTimestamp = null;
- this.specificStartupOffsets = null;
- return this;
- }
-
- /**
- * Specifies the consumer to start reading from the latest offset for all partitions. This lets
- * the consumer ignore any committed group offsets in Zookeeper / Kafka brokers.
- *
- * <p>This method does not affect where partitions are read from when the consumer is restored
- * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or savepoint,
- * only the offsets in the restored state will be used.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setStartFromLatest() {
- this.startupMode = StartupMode.LATEST;
- this.startupOffsetsTimestamp = null;
- this.specificStartupOffsets = null;
- return this;
- }
-
- /**
- * Specifies the consumer to start reading partitions from a specified timestamp. The specified
- * timestamp must be before the current timestamp. This lets the consumer ignore any committed
- * group offsets in Zookeeper / Kafka brokers.
- *
- * <p>The consumer will look up the earliest offset whose timestamp is greater than or equal to
- * the specific timestamp from Kafka. If there's no such offset, the consumer will use the
- * latest offset to read data from kafka.
- *
- * <p>This method does not affect where partitions are read from when the consumer is restored
- * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or savepoint,
- * only the offsets in the restored state will be used.
- *
- * @param startupOffsetsTimestamp timestamp for the startup offsets, as milliseconds from epoch.
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setStartFromTimestamp(long startupOffsetsTimestamp) {
- checkArgument(
- startupOffsetsTimestamp >= 0,
- "The provided value for the startup offsets timestamp is invalid.");
-
- long currentTimestamp = System.currentTimeMillis();
- checkArgument(
- startupOffsetsTimestamp <= currentTimestamp,
- "Startup time[%s] must be before current time[%s].",
- startupOffsetsTimestamp,
- currentTimestamp);
-
- this.startupMode = StartupMode.TIMESTAMP;
- this.startupOffsetsTimestamp = startupOffsetsTimestamp;
- this.specificStartupOffsets = null;
- return this;
- }
-
- /**
- * Specifies the consumer to start reading from any committed group offsets found in Zookeeper /
- * Kafka brokers. The "group.id" property must be set in the configuration properties. If no
- * offset can be found for a partition, the behaviour in "auto.offset.reset" set in the
- * configuration properties will be used for the partition.
- *
- * <p>This method does not affect where partitions are read from when the consumer is restored
- * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or savepoint,
- * only the offsets in the restored state will be used.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setStartFromGroupOffsets() {
- this.startupMode = StartupMode.GROUP_OFFSETS;
- this.startupOffsetsTimestamp = null;
- this.specificStartupOffsets = null;
- return this;
- }
-
- /**
- * Specifies the consumer to start reading partitions from specific offsets, set independently
- * for each partition. The specified offset should be the offset of the next record that will be
- * read from partitions. This lets the consumer ignore any committed group offsets in Zookeeper
- * / Kafka brokers.
- *
- * <p>If the provided map of offsets contains entries whose {@link KafkaTopicPartition} is not
- * subscribed by the consumer, the entry will be ignored. If the consumer subscribes to a
- * partition that does not exist in the provided map of offsets, the consumer will fallback to
- * the default group offset behaviour (see {@link
- * FlinkKafkaConsumerBase#setStartFromGroupOffsets()}) for that particular partition.
- *
- * <p>If the specified offset for a partition is invalid, or the behaviour for that partition is
- * defaulted to group offsets but still no group offset could be found for it, then the
- * "auto.offset.reset" behaviour set in the configuration properties will be used for the
- * partition
- *
- * <p>This method does not affect where partitions are read from when the consumer is restored
- * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or savepoint,
- * only the offsets in the restored state will be used.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> setStartFromSpecificOffsets(
- Map<KafkaTopicPartition, Long> specificStartupOffsets) {
- this.startupMode = StartupMode.SPECIFIC_OFFSETS;
- this.startupOffsetsTimestamp = null;
- this.specificStartupOffsets = checkNotNull(specificStartupOffsets);
- return this;
- }
-
- /**
- * By default, when restoring from a checkpoint / savepoint, the consumer always ignores
- * restored partitions that are no longer associated with the current specified topics or topic
- * pattern to subscribe to.
- *
- * <p>This method configures the consumer to not filter the restored partitions, therefore
- * always attempting to consume whatever partition was present in the previous execution
- * regardless of the specified topics to subscribe to in the current execution.
- *
- * @return The consumer object, to allow function chaining.
- */
- public FlinkKafkaConsumerBase<T> disableFilterRestoredPartitionsWithSubscribedTopics() {
- this.filterRestoredPartitionsWithCurrentTopicsDescriptor = false;
- return this;
- }
-
- // ------------------------------------------------------------------------
- // Work methods
- // ------------------------------------------------------------------------
-
- @Override
- public void open(Configuration configuration) throws Exception {
- // determine the offset commit mode
- this.offsetCommitMode =
- OffsetCommitModes.fromConfiguration(
- getIsAutoCommitEnabled(),
- enableCommitOnCheckpoints,
- ((StreamingRuntimeContext) getRuntimeContext()).isCheckpointingEnabled());
-
- // create the partition discoverer
- this.partitionDiscoverer =
- createPartitionDiscoverer(
- topicsDescriptor,
- getRuntimeContext().getIndexOfThisSubtask(),
- getRuntimeContext().getNumberOfParallelSubtasks());
- this.partitionDiscoverer.open();
-
- subscribedPartitionsToStartOffsets = new HashMap<>();
- final List<KafkaTopicPartition> allPartitions = partitionDiscoverer.discoverPartitions();
- if (restoredState != null) {
- for (KafkaTopicPartition partition : allPartitions) {
- if (!restoredState.containsKey(partition)) {
- restoredState.put(partition, KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);
- }
- }
-
- for (Map.Entry<KafkaTopicPartition, Long> restoredStateEntry :
- restoredState.entrySet()) {
- // seed the partition discoverer with the union state while filtering out
- // restored partitions that should not be subscribed by this subtask
- if (KafkaTopicPartitionAssigner.assign(
- restoredStateEntry.getKey(),
- getRuntimeContext().getNumberOfParallelSubtasks())
- == getRuntimeContext().getIndexOfThisSubtask()) {
- subscribedPartitionsToStartOffsets.put(
- restoredStateEntry.getKey(), restoredStateEntry.getValue());
- }
- }
-
- if (filterRestoredPartitionsWithCurrentTopicsDescriptor) {
- subscribedPartitionsToStartOffsets
- .entrySet()
- .removeIf(
- entry -> {
- if (!topicsDescriptor.isMatchingTopic(
- entry.getKey().getTopic())) {
- LOG.warn(
- "{} is removed from subscribed partitions since it is no longer associated with topics descriptor of current execution.",
- entry.getKey());
- return true;
- }
- return false;
- });
- }
-
- LOG.info(
- "Consumer subtask {} will start reading {} partitions with offsets in restored state: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- subscribedPartitionsToStartOffsets);
- } else {
- // use the partition discoverer to fetch the initial seed partitions,
- // and set their initial offsets depending on the startup mode.
- // for SPECIFIC_OFFSETS and TIMESTAMP modes, we set the specific offsets now;
- // for other modes (EARLIEST, LATEST, and GROUP_OFFSETS), the offset is lazily
- // determined
- // when the partition is actually read.
- switch (startupMode) {
- case SPECIFIC_OFFSETS:
- if (specificStartupOffsets == null) {
- throw new IllegalStateException(
- "Startup mode for the consumer set to "
- + StartupMode.SPECIFIC_OFFSETS
- + ", but no specific offsets were specified.");
- }
-
- for (KafkaTopicPartition seedPartition : allPartitions) {
- Long specificOffset = specificStartupOffsets.get(seedPartition);
- if (specificOffset != null) {
- // since the specified offsets represent the next record to read, we
- // subtract
- // it by one so that the initial state of the consumer will be correct
- subscribedPartitionsToStartOffsets.put(
- seedPartition, specificOffset - 1);
- } else {
- // default to group offset behaviour if the user-provided specific
- // offsets
- // do not contain a value for this partition
- subscribedPartitionsToStartOffsets.put(
- seedPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
- }
- }
-
- break;
- case TIMESTAMP:
- if (startupOffsetsTimestamp == null) {
- throw new IllegalStateException(
- "Startup mode for the consumer set to "
- + StartupMode.TIMESTAMP
- + ", but no startup timestamp was specified.");
- }
-
- for (Map.Entry<KafkaTopicPartition, Long> partitionToOffset :
- fetchOffsetsWithTimestamp(allPartitions, startupOffsetsTimestamp)
- .entrySet()) {
- subscribedPartitionsToStartOffsets.put(
- partitionToOffset.getKey(),
- (partitionToOffset.getValue() == null)
- // if an offset cannot be retrieved for a partition with the
- // given timestamp,
- // we default to using the latest offset for the partition
- ? KafkaTopicPartitionStateSentinel.LATEST_OFFSET
- // since the specified offsets represent the next record to
- // read, we subtract
- // it by one so that the initial state of the consumer will
- // be correct
- : partitionToOffset.getValue() - 1);
- }
-
- break;
- default:
- for (KafkaTopicPartition seedPartition : allPartitions) {
- subscribedPartitionsToStartOffsets.put(
- seedPartition, startupMode.getStateSentinel());
- }
- }
-
- if (!subscribedPartitionsToStartOffsets.isEmpty()) {
- switch (startupMode) {
- case EARLIEST:
- LOG.info(
- "Consumer subtask {} will start reading the following {} partitions from the earliest offsets: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- subscribedPartitionsToStartOffsets.keySet());
- break;
- case LATEST:
- LOG.info(
- "Consumer subtask {} will start reading the following {} partitions from the latest offsets: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- subscribedPartitionsToStartOffsets.keySet());
- break;
- case TIMESTAMP:
- LOG.info(
- "Consumer subtask {} will start reading the following {} partitions from timestamp {}: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- startupOffsetsTimestamp,
- subscribedPartitionsToStartOffsets.keySet());
- break;
- case SPECIFIC_OFFSETS:
- LOG.info(
- "Consumer subtask {} will start reading the following {} partitions from the specified startup offsets {}: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- specificStartupOffsets,
- subscribedPartitionsToStartOffsets.keySet());
-
- List<KafkaTopicPartition> partitionsDefaultedToGroupOffsets =
- new ArrayList<>(subscribedPartitionsToStartOffsets.size());
- for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition :
- subscribedPartitionsToStartOffsets.entrySet()) {
- if (subscribedPartition.getValue()
- == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) {
- partitionsDefaultedToGroupOffsets.add(subscribedPartition.getKey());
- }
- }
-
- if (partitionsDefaultedToGroupOffsets.size() > 0) {
- LOG.warn(
- "Consumer subtask {} cannot find offsets for the following {} partitions in the specified startup offsets: {}"
- + "; their startup offsets will be defaulted to their committed group offsets in Kafka.",
- getRuntimeContext().getIndexOfThisSubtask(),
- partitionsDefaultedToGroupOffsets.size(),
- partitionsDefaultedToGroupOffsets);
- }
- break;
- case GROUP_OFFSETS:
- LOG.info(
- "Consumer subtask {} will start reading the following {} partitions from the committed group offsets in Kafka: {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets.size(),
- subscribedPartitionsToStartOffsets.keySet());
- }
- } else {
- LOG.info(
- "Consumer subtask {} initially has no partitions to read from.",
- getRuntimeContext().getIndexOfThisSubtask());
- }
- }
-
- this.deserializer.open(
- RuntimeContextInitializationContextAdapters.deserializationAdapter(
- getRuntimeContext(), metricGroup -> metricGroup.addGroup("user")));
- }
-
- @Override
- public void run(SourceContext<T> sourceContext) throws Exception {
- if (subscribedPartitionsToStartOffsets == null) {
- throw new Exception("The partitions were not set for the consumer");
- }
-
- // initialize commit metrics and default offset callback method
- this.successfulCommits =
- this.getRuntimeContext()
- .getMetricGroup()
- .counter(COMMITS_SUCCEEDED_METRICS_COUNTER);
- this.failedCommits =
- this.getRuntimeContext().getMetricGroup().counter(COMMITS_FAILED_METRICS_COUNTER);
- final int subtaskIndex = this.getRuntimeContext().getIndexOfThisSubtask();
-
- this.offsetCommitCallback =
- new KafkaCommitCallback() {
- @Override
- public void onSuccess() {
- successfulCommits.inc();
- }
-
- @Override
- public void onException(Throwable cause) {
- LOG.warn(
- String.format(
- "Consumer subtask %d failed async Kafka commit.",
- subtaskIndex),
- cause);
- failedCommits.inc();
- }
- };
-
- // mark the subtask as temporarily idle if there are no initial seed partitions;
- // once this subtask discovers some partitions and starts collecting records, the subtask's
- // status will automatically be triggered back to be active.
- if (subscribedPartitionsToStartOffsets.isEmpty()) {
- sourceContext.markAsTemporarilyIdle();
- }
-
- LOG.info(
- "Consumer subtask {} creating fetcher with offsets {}.",
- getRuntimeContext().getIndexOfThisSubtask(),
- subscribedPartitionsToStartOffsets);
- // from this point forward:
- // - 'snapshotState' will draw offsets from the fetcher,
- // instead of being built from `subscribedPartitionsToStartOffsets`
- // - 'notifyCheckpointComplete' will start to do work (i.e. commit offsets to
- // Kafka through the fetcher, if configured to do so)
- this.kafkaFetcher =
- createFetcher(
- sourceContext,
- subscribedPartitionsToStartOffsets,
- watermarkStrategy,
- (StreamingRuntimeContext) getRuntimeContext(),
- offsetCommitMode,
- getRuntimeContext().getMetricGroup().addGroup(KAFKA_CONSUMER_METRICS_GROUP),
- useMetrics);
-
- if (!running) {
- return;
- }
-
- // depending on whether we were restored with the current state version (1.3),
- // remaining logic branches off into 2 paths:
- // 1) New state - partition discovery loop executed as separate thread, with this
- // thread running the main fetcher loop
- // 2) Old state - partition discovery is disabled and only the main fetcher loop is
- // executed
- if (discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED) {
- kafkaFetcher.runFetchLoop();
- } else {
- runWithPartitionDiscovery();
- }
- }
-
- private void runWithPartitionDiscovery() throws Exception {
- final AtomicReference<Exception> discoveryLoopErrorRef = new AtomicReference<>();
- createAndStartDiscoveryLoop(discoveryLoopErrorRef);
-
- kafkaFetcher.runFetchLoop();
-
- // make sure that the partition discoverer is waked up so that
- // the discoveryLoopThread exits
- partitionDiscoverer.wakeup();
- joinDiscoveryLoopThread();
-
- // rethrow any fetcher errors
- final Exception discoveryLoopError = discoveryLoopErrorRef.get();
- if (discoveryLoopError != null) {
- throw new RuntimeException(discoveryLoopError);
- }
- }
-
- @VisibleForTesting
- void joinDiscoveryLoopThread() throws InterruptedException {
- if (discoveryLoopThread != null) {
- discoveryLoopThread.join();
- }
- }
-
- private void createAndStartDiscoveryLoop(AtomicReference<Exception> discoveryLoopErrorRef) {
- discoveryLoopThread =
- new Thread(
- () -> {
- try {
- // --------------------- partition discovery loop
- // ---------------------
-
- // throughout the loop, we always eagerly check if we are still
- // running before
- // performing the next operation, so that we can escape the loop as
- // soon as possible
-
- while (running) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Consumer subtask {} is trying to discover new partitions ...",
- getRuntimeContext().getIndexOfThisSubtask());
- }
-
- final List<KafkaTopicPartition> discoveredPartitions;
- try {
- discoveredPartitions =
- partitionDiscoverer.discoverPartitions();
- } catch (AbstractPartitionDiscoverer.WakeupException
- | AbstractPartitionDiscoverer.ClosedException e) {
- // the partition discoverer may have been closed or woken up
- // before or during the discovery;
- // this would only happen if the consumer was canceled;
- // simply escape the loop
- break;
- }
-
- // no need to add the discovered partitions if we were closed
- // during the meantime
- if (running && !discoveredPartitions.isEmpty()) {
- kafkaFetcher.addDiscoveredPartitions(discoveredPartitions);
- }
-
- // do not waste any time sleeping if we're not running anymore
- if (running && discoveryIntervalMillis != 0) {
- try {
- Thread.sleep(discoveryIntervalMillis);
- } catch (InterruptedException iex) {
- // may be interrupted if the consumer was canceled
- // midway; simply escape the loop
- break;
- }
- }
- }
- } catch (Exception e) {
- discoveryLoopErrorRef.set(e);
- } finally {
- // calling cancel will also let the fetcher loop escape
- // (if not running, cancel() was already called)
- if (running) {
- cancel();
- }
- }
- },
- "Kafka Partition Discovery for "
- + getRuntimeContext().getTaskNameWithSubtasks());
-
- discoveryLoopThread.start();
- }
-
- @Override
- public void cancel() {
- // set ourselves as not running;
- // this would let the main discovery loop escape as soon as possible
- running = false;
-
- if (discoveryLoopThread != null) {
-
- if (partitionDiscoverer != null) {
- // we cannot close the discoverer here, as it is error-prone to concurrent access;
- // only wakeup the discoverer, the discovery loop will clean itself up after it
- // escapes
- partitionDiscoverer.wakeup();
- }
-
- // the discovery loop may currently be sleeping in-between
- // consecutive discoveries; interrupt to shutdown faster
- discoveryLoopThread.interrupt();
- }
-
- // abort the fetcher, if there is one
- if (kafkaFetcher != null) {
- kafkaFetcher.cancel();
- }
- }
-
- @Override
- public void close() throws Exception {
- cancel();
-
- joinDiscoveryLoopThread();
-
- Exception exception = null;
- if (partitionDiscoverer != null) {
- try {
- partitionDiscoverer.close();
- } catch (Exception e) {
- exception = e;
- }
- }
-
- try {
- super.close();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
-
- if (exception != null) {
- throw exception;
- }
- }
-
- // ------------------------------------------------------------------------
- // Checkpoint and restore
- // ------------------------------------------------------------------------
-
- @Override
- public final void initializeState(FunctionInitializationContext context) throws Exception {
-
- OperatorStateStore stateStore = context.getOperatorStateStore();
-
- this.unionOffsetStates =
- stateStore.getUnionListState(
- new ListStateDescriptor<>(
- OFFSETS_STATE_NAME,
- createStateSerializer(getRuntimeContext().getExecutionConfig())));
-
- if (context.isRestored()) {
- restoredState = new TreeMap<>(new KafkaTopicPartition.Comparator());
-
- // populate actual holder for restored state
- for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : unionOffsetStates.get()) {
- restoredState.put(kafkaOffset.f0, kafkaOffset.f1);
- }
-
- LOG.info(
- "Consumer subtask {} restored state: {}.",
- getRuntimeContext().getIndexOfThisSubtask(),
- restoredState);
- } else {
- LOG.info(
- "Consumer subtask {} has no restore state.",
- getRuntimeContext().getIndexOfThisSubtask());
- }
- }
-
- @Override
- public final void snapshotState(FunctionSnapshotContext context) throws Exception {
- if (!running) {
- LOG.debug("snapshotState() called on closed source");
- } else {
- unionOffsetStates.clear();
-
- final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
- if (fetcher == null) {
- // the fetcher has not yet been initialized, which means we need to return the
- // originally restored offsets or the assigned partitions
- for (Map.Entry<KafkaTopicPartition, Long> subscribedPartition :
- subscribedPartitionsToStartOffsets.entrySet()) {
- unionOffsetStates.add(
- Tuple2.of(
- subscribedPartition.getKey(), subscribedPartition.getValue()));
- }
-
- if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
- // the map cannot be asynchronously updated, because only one checkpoint call
- // can happen
- // on this function at a time: either snapshotState() or
- // notifyCheckpointComplete()
- pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState);
- }
- } else {
- HashMap<KafkaTopicPartition, Long> currentOffsets = fetcher.snapshotCurrentState();
-
- if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
- // the map cannot be asynchronously updated, because only one checkpoint call
- // can happen
- // on this function at a time: either snapshotState() or
- // notifyCheckpointComplete()
- pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
- }
-
- for (Map.Entry<KafkaTopicPartition, Long> kafkaTopicPartitionLongEntry :
- currentOffsets.entrySet()) {
- unionOffsetStates.add(
- Tuple2.of(
- kafkaTopicPartitionLongEntry.getKey(),
- kafkaTopicPartitionLongEntry.getValue()));
- }
- }
-
- if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
- // truncate the map of pending offsets to commit, to prevent infinite growth
- while (pendingOffsetsToCommit.size() > MAX_NUM_PENDING_CHECKPOINTS) {
- pendingOffsetsToCommit.remove(0);
- }
- }
- }
- }
-
- @Override
- public final void notifyCheckpointComplete(long checkpointId) throws Exception {
- if (!running) {
- LOG.debug("notifyCheckpointComplete() called on closed source");
- return;
- }
-
- final AbstractFetcher<?, ?> fetcher = this.kafkaFetcher;
- if (fetcher == null) {
- LOG.debug("notifyCheckpointComplete() called on uninitialized source");
- return;
- }
-
- if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS) {
- // only one commit operation must be in progress
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Consumer subtask {} committing offsets to Kafka/ZooKeeper for checkpoint {}.",
- getRuntimeContext().getIndexOfThisSubtask(),
- checkpointId);
- }
-
- try {
- final int posInMap = pendingOffsetsToCommit.indexOf(checkpointId);
- if (posInMap == -1) {
- LOG.warn(
- "Consumer subtask {} received confirmation for unknown checkpoint id {}",
- getRuntimeContext().getIndexOfThisSubtask(),
- checkpointId);
- return;
- }
-
- @SuppressWarnings("unchecked")
- Map<KafkaTopicPartition, Long> offsets =
- (Map<KafkaTopicPartition, Long>) pendingOffsetsToCommit.remove(posInMap);
-
- // remove older checkpoints in map
- for (int i = 0; i < posInMap; i++) {
- pendingOffsetsToCommit.remove(0);
- }
-
- if (offsets == null || offsets.size() == 0) {
- LOG.debug(
- "Consumer subtask {} has empty checkpoint state.",
- getRuntimeContext().getIndexOfThisSubtask());
- return;
- }
-
- fetcher.commitInternalOffsetsToKafka(offsets, offsetCommitCallback);
- } catch (Exception e) {
- if (running) {
- throw e;
- }
- // else ignore exception if we are no longer running
- }
- }
- }
-
- @Override
- public void notifyCheckpointAborted(long checkpointId) {}
-
- // ------------------------------------------------------------------------
- // Kafka Consumer specific methods
- // ------------------------------------------------------------------------
-
- /**
- * Creates the fetcher that connect to the Kafka brokers, pulls data, deserialized the data, and
- * emits it into the data streams.
- *
- * @param sourceContext The source context to emit data to.
- * @param subscribedPartitionsToStartOffsets The set of partitions that this subtask should
- * handle, with their start offsets.
- * @param watermarkStrategy Optional, a serialized WatermarkStrategy.
- * @param runtimeContext The task's runtime context.
- * @return The instantiated fetcher
- * @throws Exception The method should forward exceptions
- */
- protected abstract AbstractFetcher<T, ?> createFetcher(
- SourceContext<T> sourceContext,
- Map<KafkaTopicPartition, Long> subscribedPartitionsToStartOffsets,
- SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
- StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode,
- MetricGroup kafkaMetricGroup,
- boolean useMetrics)
- throws Exception;
-
- /**
- * Creates the partition discoverer that is used to find new partitions for this subtask.
- *
- * @param topicsDescriptor Descriptor that describes whether we are discovering partitions for
- * fixed topics or a topic pattern.
- * @param indexOfThisSubtask The index of this consumer subtask.
- * @param numParallelSubtasks The total number of parallel consumer subtasks.
- * @return The instantiated partition discoverer
- */
- protected abstract AbstractPartitionDiscoverer createPartitionDiscoverer(
- KafkaTopicsDescriptor topicsDescriptor,
- int indexOfThisSubtask,
- int numParallelSubtasks);
-
- protected abstract boolean getIsAutoCommitEnabled();
-
- protected abstract Map<KafkaTopicPartition, Long> fetchOffsetsWithTimestamp(
- Collection<KafkaTopicPartition> partitions, long timestamp);
-
- // ------------------------------------------------------------------------
- // ResultTypeQueryable methods
- // ------------------------------------------------------------------------
-
- @Override
- public TypeInformation<T> getProducedType() {
- return deserializer.getProducedType();
- }
-
- // ------------------------------------------------------------------------
- // Test utilities
- // ------------------------------------------------------------------------
-
- @VisibleForTesting
- Map<KafkaTopicPartition, Long> getSubscribedPartitionsToStartOffsets() {
- return subscribedPartitionsToStartOffsets;
- }
-
- @VisibleForTesting
- TreeMap<KafkaTopicPartition, Long> getRestoredState() {
- return restoredState;
- }
-
- @VisibleForTesting
- OffsetCommitMode getOffsetCommitMode() {
- return offsetCommitMode;
- }
-
- @VisibleForTesting
- LinkedMap getPendingOffsetsToCommit() {
- return pendingOffsetsToCommit;
- }
-
- @VisibleForTesting
- public boolean getEnableCommitOnCheckpoints() {
- return enableCommitOnCheckpoints;
- }
-
- /**
- * Creates state serializer for kafka topic partition to offset tuple. Using of the explicit
- * state serializer with KryoSerializer is needed because otherwise users cannot use
- * 'disableGenericTypes' properties with KafkaConsumer.
- */
- @VisibleForTesting
- static TupleSerializer<Tuple2<KafkaTopicPartition, Long>> createStateSerializer(
- ExecutionConfig executionConfig) {
- // explicit serializer will keep the compatibility with GenericTypeInformation and allow to
- // disableGenericTypes for users
- TypeSerializer<?>[] fieldSerializers =
- new TypeSerializer<?>[] {
- new KryoSerializer<>(KafkaTopicPartition.class, executionConfig),
- LongSerializer.INSTANCE
- };
- @SuppressWarnings("unchecked")
- Class<Tuple2<KafkaTopicPartition, Long>> tupleClass =
- (Class<Tuple2<KafkaTopicPartition, Long>>) (Class<?>) Tuple2.class;
- return new TupleSerializer<>(tupleClass, fieldSerializers);
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaErrorCode.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaErrorCode.java
deleted file mode 100644
index c11f1b1768f..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaErrorCode.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-/** Error codes used in {@link FlinkKafkaException}. */
-@PublicEvolving
-public enum FlinkKafkaErrorCode {
- PRODUCERS_POOL_EMPTY,
- EXTERNAL_ERROR
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaException.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaException.java
deleted file mode 100644
index 77d02313028..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.util.FlinkException;
-
-/** Exception used by {@link FlinkKafkaProducer} and {@link FlinkKafkaConsumer}. */
-@PublicEvolving
-public class FlinkKafkaException extends FlinkException {
-
- private static final long serialVersionUID = 920269130311214200L;
-
- private final FlinkKafkaErrorCode errorCode;
-
- public FlinkKafkaException(FlinkKafkaErrorCode errorCode, String message) {
- super(message);
- this.errorCode = errorCode;
- }
-
- public FlinkKafkaException(FlinkKafkaErrorCode errorCode, String message, Throwable cause) {
- super(message, cause);
- this.errorCode = errorCode;
- }
-
- public FlinkKafkaErrorCode getErrorCode() {
- return errorCode;
- }
-}
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
deleted file mode 100644
index ca81cc8e391..00000000000
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java
+++ /dev/null
@@ -1,1950 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.serialization.RuntimeContextInitializationContextAdapters;
-import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.api.common.state.ListState;
-import org.apache.flink.api.common.state.ListStateDescriptor;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.SimpleTypeSerializerSnapshot;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
-import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.runtime.state.FunctionInitializationContext;
-import org.apache.flink.runtime.state.FunctionSnapshotContext;
-import org.apache.flink.streaming.api.functions.sink.TwoPhaseCommitSinkFunction;
-import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
-import org.apache.flink.streaming.connectors.kafka.internals.FlinkKafkaInternalProducer;
-import org.apache.flink.streaming.connectors.kafka.internals.KafkaSerializationSchemaWrapper;
-import org.apache.flink.streaming.connectors.kafka.internals.TransactionalIdsGenerator;
-import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricMutableWrapper;
-import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner;
-import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner;
-import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.NetUtils;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.TemporaryClassLoaderContext;
-
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kafka.clients.producer.Callback;
-import org.apache.kafka.clients.producer.Producer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.common.Metric;
-import org.apache.kafka.common.MetricName;
-import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.errors.InvalidTxnStateException;
-import org.apache.kafka.common.errors.ProducerFencedException;
-import org.apache.kafka.common.serialization.ByteArraySerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * Flink Sink to produce data into a Kafka topic. By default producer will use {@link
- * FlinkKafkaProducer.Semantic#AT_LEAST_ONCE} semantic. Before using {@link
- * FlinkKafkaProducer.Semantic#EXACTLY_ONCE} please refer to Flink's Kafka connector documentation.
- *
- * @deprecated Please use {@link org.apache.flink.connector.kafka.sink.KafkaSink}.
- */
-@Deprecated
-@PublicEvolving
-public class FlinkKafkaProducer<IN>
- extends TwoPhaseCommitSinkFunction<
- IN,
- FlinkKafkaProducer.KafkaTransactionState,
- FlinkKafkaProducer.KafkaTransactionContext> {
-
- /**
- * Semantics that can be chosen.
- * <li>{@link #EXACTLY_ONCE}
- * <li>{@link #AT_LEAST_ONCE}
- * <li>{@link #NONE}
- */
- public enum Semantic {
-
- /**
- * Semantic.EXACTLY_ONCE the Flink producer will write all messages in a Kafka transaction
- * that will be committed to Kafka on a checkpoint.
- *
- * <p>In this mode {@link FlinkKafkaProducer} sets up a pool of {@link
- * FlinkKafkaInternalProducer}. Between each checkpoint a Kafka transaction is created,
- * which is committed on {@link FlinkKafkaProducer#notifyCheckpointComplete(long)}. If
- * checkpoint complete notifications are running late, {@link FlinkKafkaProducer} can run
- * out of {@link FlinkKafkaInternalProducer}s in the pool. In that case any subsequent
- * {@link FlinkKafkaProducer#snapshotState(FunctionSnapshotContext)} requests will fail and
- * {@link FlinkKafkaProducer} will keep using the {@link FlinkKafkaInternalProducer} from
- * the previous checkpoint. To decrease the chance of failing checkpoints there are four
- * options:
- * <li>decrease number of max concurrent checkpoints
- * <li>make checkpoints more reliable (so that they complete faster)
- * <li>increase the delay between checkpoints
- * <li>increase the size of {@link FlinkKafkaInternalProducer}s pool
- */
- EXACTLY_ONCE,
-
- /**
- * Semantic.AT_LEAST_ONCE the Flink producer will wait for all outstanding messages in the
- * Kafka buffers to be acknowledged by the Kafka producer on a checkpoint.
- */
- AT_LEAST_ONCE,
-
- /**
- * Semantic.NONE means that nothing will be guaranteed. Messages can be lost and/or
- * duplicated in case of failure.
- */
- NONE
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(FlinkKafkaProducer.class);
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Number of characters to truncate the taskName to for the Kafka transactionalId. The maximum
- * this can possibly be set to is 32,767 - (length of operatorUniqueId).
- */
- private static final short maxTaskNameSize = 1_000;
-
- /**
- * This coefficient determines what is the safe scale down factor.
- *
- * <p>If the Flink application previously failed before first checkpoint completed or we are
- * starting new batch of {@link FlinkKafkaProducer} from scratch without clean shutdown of the
- * previous one, {@link FlinkKafkaProducer} doesn't know what was the set of previously used
- * Kafka's transactionalId's. In that case, it will try to play safe and abort all of the
- * possible transactionalIds from the range of: {@code [0, getNumberOfParallelSubtasks() *
- * kafkaProducersPoolSize * SAFE_SCALE_DOWN_FACTOR) }
- *
- * <p>The range of available to use transactional ids is: {@code [0,
- * getNumberOfParallelSubtasks() * kafkaProducersPoolSize) }
- *
- * <p>This means that if we decrease {@code getNumberOfParallelSubtasks()} by a factor larger
- * than {@code SAFE_SCALE_DOWN_FACTOR} we can have a left some lingering transaction.
- */
- public static final int SAFE_SCALE_DOWN_FACTOR = 5;
-
- /**
- * Default number of KafkaProducers in the pool. See {@link
- * FlinkKafkaProducer.Semantic#EXACTLY_ONCE}.
- */
- public static final int DEFAULT_KAFKA_PRODUCERS_POOL_SIZE = 5;
-
- /** Default value for kafka transaction timeout. */
- public static final Time DEFAULT_KAFKA_TRANSACTION_TIMEOUT = Time.hours(1);
-
- /** Configuration key for disabling the metrics reporting. */
- public static final String KEY_DISABLE_METRICS = "flink.disable-metrics";
-
- /**
- * Descriptor of the transactional IDs list. Note: This state is serialized by Kryo Serializer
- * and it has compatibility problem that will be removed later. Please use
- * NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2.
- */
- @Deprecated
- private static final ListStateDescriptor<FlinkKafkaProducer.NextTransactionalIdHint>
- NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR =
- new ListStateDescriptor<>(
- "next-transactional-id-hint",
- TypeInformation.of(NextTransactionalIdHint.class));
-
- private static final ListStateDescriptor<FlinkKafkaProducer.NextTransactionalIdHint>
- NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2 =
- new ListStateDescriptor<>(
- "next-transactional-id-hint-v2",
- new NextTransactionalIdHintSerializer());
-
- /** State for nextTransactionalIdHint. */
- private transient ListState<FlinkKafkaProducer.NextTransactionalIdHint>
- nextTransactionalIdHintState;
-
- /** Generator for Transactional IDs. */
- private transient TransactionalIdsGenerator transactionalIdsGenerator;
-
- /** Hint for picking next transactional id. */
- private transient FlinkKafkaProducer.NextTransactionalIdHint nextTransactionalIdHint;
-
- /** User defined properties for the Producer. */
- protected final Properties producerConfig;
-
- /** The name of the default topic this producer is writing data to. */
- protected final String defaultTopicId;
-
- /**
- * (Serializable) SerializationSchema for turning objects used with Flink into. byte[] for
- * Kafka.
- */
- @Nullable private final KeyedSerializationSchema<IN> keyedSchema;
-
- /**
- * (Serializable) serialization schema for serializing records to {@link ProducerRecord
... 43391 lines suppressed ...