You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/02/15 08:33:22 UTC
[GitHub] [flink] leonardBang commented on a change in pull request #18496: [FLINK-25289][tests] add sink test suite in connector testframe
leonardBang commented on a change in pull request #18496:
URL: https://github.com/apache/flink/pull/18496#discussion_r805352388
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaDataReader.java
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.WakeupException;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+/** Kafka dataStream data reader. */
+public class KafkaDataReader implements ExternalSystemDataReader<String> {
+ private final KafkaConsumer<String, String> consumer;
+
+ public KafkaDataReader(Properties properties, Collection<TopicPartition> partitions) {
+ this.consumer = new KafkaConsumer<>(properties);
+ consumer.assign(partitions);
+ consumer.seekToBeginning(partitions);
+ }
+
+ @Override
+ public List<String> poll(Duration timeout) {
+ List<String> result = new LinkedList<>();
+ ConsumerRecords<String, String> consumerRecords;
+ try {
+ consumerRecords = consumer.poll(timeout);
+ } catch (WakeupException we) {
+ return Collections.emptyList();
+ }
+ Iterator<ConsumerRecord<String, String>> iterator = consumerRecords.iterator();
+ while (iterator.hasNext()) {
+ result.add(iterator.next().value());
+ }
+ return result;
+ }
+
+ @Override
+ public void close() throws Exception {
+ consumer.close();
Review comment:
hint: check null before release resource
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
Review comment:
```suggestion
private static final long DEFAULT_TIMEOUT = 30L;
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
Review comment:
```suggestion
final Properties config = new Properties();
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
+ config.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ return AdminClient.create(config);
+ }
+
+ @Override
+ public Sink<String> createSink(TestingSinkSettings sinkSettings) {
+ if (!topicExists(topicName)) {
+ createTopic(topicName, 4, (short) 1);
+ }
+
+ KafkaSinkBuilder<String> builder = KafkaSink.builder();
+ Properties properties = new Properties();
+ properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 900000);
Review comment:
```suggestion
final Properties properties = new Properties();
properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_TRANSACTION_TIMEOUT_IN_MS);
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSource.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumerator;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.util.List;
+
+/**
+ * The source reads data from a list and stops reading at the fixed position. The source will wait
+ * until the checkpoint or savepoint triggers.
+ *
+ * <p>Note that this source must be of parallelism 1.
+ */
+public class FromElementsSource<OUT> implements Source<OUT, FromElementsSplit, NoOpEnumState> {
+ // Boundedness
+ private Boundedness boundedness;
+
+ private List<OUT> elements;
+
+ private Integer successNum;
+
+ public FromElementsSource(List<OUT> elements) {
+ this.elements = elements;
+ }
+
+ public FromElementsSource(Boundedness boundedness, List<OUT> elements, Integer successNum) {
+ this(elements);
+ if (successNum > elements.size()) {
+ throw new RuntimeException("SuccessNum must be larger than elements' size.");
Review comment:
```suggestion
Preconditions.check(successNum > elements.size(), String.format("The successNum must be larger than the elements list %d, but actual successNum is %d", elements.size(), successNum));
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSource.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumerator;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.util.List;
+
+/**
+ * The source reads data from a list and stops reading at the fixed position. The source will wait
+ * until the checkpoint or savepoint triggers.
Review comment:
```suggestion
* A {@link Source} implementation that reads data from a list and stops reading at the fixed position.
* The source will wait until the checkpoint or savepoint triggered, the source is useful for connector tests.
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/split/FromElementsSplit.java
##########
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source.split;
+
+import org.apache.flink.api.connector.source.SourceSplit;
+
+/** The split of the list source. */
Review comment:
```suggestion
/** The split of {@link FromElementsSource}. */
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a higher parallelism 4.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a higher parallelism")
+ public void testScaleUp(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 4);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a lower parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 4 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a lower parallelism 2.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a lower parallelism")
+ public void testScaleDown(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 4, 2);
+ }
+
+ private void restartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic,
+ final int beforeParallelism,
+ final int afterParallelism)
+ throws Exception {
+ // Step 1: Preparation
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.setRestartStrategy(RestartStrategies.noRestart());
+
+ // Step 2: Generate test data
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Step 3: Build and execute Flink job
+ int numBeforeSuccess = testRecords.size() / 2;
+ DataStreamSource<T> source =
+ execEnv.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ numBeforeSuccess),
+ WatermarkStrategy.noWatermarks(),
+ "beforeRestartSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name("Sink restart test")
+ .setParallelism(beforeParallelism);
+
+ /**
+ * The job should stop after consume a specified number of records. In order to know when
+ * the specified number of records have been consumed, a collect sink is need to be watched.
+ */
+ CollectResultIterator<T> iterator = addCollectSink(source);
+ final JobClient jobClient = execEnv.executeAsync("Restart Test");
+ iterator.setJobClient(jobClient);
+
+ // Step 4: Wait for the expected result and stop Flink job with a savepoint
+ String savepointDir;
+ try {
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitExpectedSizeData(iterator, numBeforeSuccess);
+
+ savepointDir =
+ jobClient
+ .stopWithSavepoint(
+ true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL)
+ .get(30, TimeUnit.SECONDS);
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+ } catch (Exception e) {
+ killJob(jobClient);
+ throw e;
+ }
+
+ List<T> target = testRecords.subList(0, numBeforeSuccess);
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), target, semantic);
+
+ // Step 4: restart the Flink job with the savepoint
+ final StreamExecutionEnvironment restartEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .setSavepointRestorePath(savepointDir)
+ .build());
+ restartEnv.enableCheckpointing(50);
+
+ DataStreamSource<T> restartSource =
+ restartEnv
+ .fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "restartSource")
+ .setParallelism(1);
+
+ DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
+ tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
+ addCollectSink(restartSource);
+ final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
+
+ try {
+ // Check the result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ } finally {
+ killJob(restartJobClient);
+ iterator.close();
+ }
+ }
+
+ /**
+ * Test connector sink metrics.
+ *
+ * <p>This test will create a sink in the external system, generate test data and write them to
+ * the sink via a Flink job. Then read and compare the metrics.
+ *
+ * <p>Now test: numRecordsOut
+ */
+ @TestTemplate
+ @DisplayName("Test sink metrics")
+ public void testMetrics(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ int parallelism = 1;
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // make sure use different names when executes multi times
+ String sinkName = "metricTestSink" + testRecords.hashCode();
+ final StreamExecutionEnvironment env =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ env.enableCheckpointing(50);
+
+ DataStreamSource<T> source =
+ env.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "metricTestSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name(sinkName)
+ .setParallelism(parallelism);
+ final JobClient jobClient = env.executeAsync("Metrics Test");
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ try {
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitUntilCondition(
+ () -> {
+ // test metrics
+ try {
+ return compareSinkMetrics(
+ queryRestClient,
+ testEnv,
+ externalContext,
+ jobClient.getJobID(),
+ sinkName,
+ testRecords.size());
+ } catch (Exception e) {
+ // skip failed assert try
+ return false;
+ }
+ },
+ Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
+ } finally {
+ // Clean up
+ killJob(jobClient);
+ }
+ }
+
+ // ----------------------------- Helper Functions ---------------------------------
+
+ /**
+ * Generate a set of test records.
+ *
+ * @param testingSinkSettings sink settings
+ * @param externalContext External context
+ * @return Collection of generated test records
+ */
+ protected List<T> generateTestData(
+ TestingSinkSettings testingSinkSettings,
+ DataStreamSinkExternalContext<T> externalContext) {
+ return externalContext.generateTestData(
+ testingSinkSettings, ThreadLocalRandom.current().nextLong());
+ }
+
+ /**
+ * Poll records from the sink.
+ *
+ * @param result Append records to which list
+ * @param reader The sink reader
+ * @param expected The expected list which help to stop polling
+ * @param retryTimes The retry times
+ * @param semantic The semantic
+ * @return Collection of records in the Sink
+ */
+ private List<T> appendResultData(
Review comment:
The method name is different with the method java document
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a higher parallelism 4.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a higher parallelism")
+ public void testScaleUp(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 4);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a lower parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 4 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a lower parallelism 2.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a lower parallelism")
+ public void testScaleDown(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 4, 2);
+ }
+
+ private void restartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic,
+ final int beforeParallelism,
+ final int afterParallelism)
+ throws Exception {
+ // Step 1: Preparation
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.setRestartStrategy(RestartStrategies.noRestart());
+
+ // Step 2: Generate test data
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Step 3: Build and execute Flink job
+ int numBeforeSuccess = testRecords.size() / 2;
+ DataStreamSource<T> source =
+ execEnv.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ numBeforeSuccess),
+ WatermarkStrategy.noWatermarks(),
+ "beforeRestartSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name("Sink restart test")
+ .setParallelism(beforeParallelism);
+
+ /**
+ * The job should stop after consume a specified number of records. In order to know when
+ * the specified number of records have been consumed, a collect sink is need to be watched.
+ */
+ CollectResultIterator<T> iterator = addCollectSink(source);
+ final JobClient jobClient = execEnv.executeAsync("Restart Test");
+ iterator.setJobClient(jobClient);
+
+ // Step 4: Wait for the expected result and stop Flink job with a savepoint
+ String savepointDir;
+ try {
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitExpectedSizeData(iterator, numBeforeSuccess);
+
+ savepointDir =
+ jobClient
+ .stopWithSavepoint(
+ true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL)
+ .get(30, TimeUnit.SECONDS);
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+ } catch (Exception e) {
+ killJob(jobClient);
+ throw e;
+ }
+
+ List<T> target = testRecords.subList(0, numBeforeSuccess);
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), target, semantic);
+
+ // Step 4: restart the Flink job with the savepoint
+ final StreamExecutionEnvironment restartEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .setSavepointRestorePath(savepointDir)
+ .build());
+ restartEnv.enableCheckpointing(50);
+
+ DataStreamSource<T> restartSource =
+ restartEnv
+ .fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "restartSource")
+ .setParallelism(1);
+
+ DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
+ tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
+ addCollectSink(restartSource);
+ final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
+
+ try {
+ // Check the result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ } finally {
+ killJob(restartJobClient);
+ iterator.close();
+ }
+ }
+
+ /**
+ * Test connector sink metrics.
+ *
+ * <p>This test will create a sink in the external system, generate test data and write them to
+ * the sink via a Flink job. Then read and compare the metrics.
+ *
+ * <p>Now test: numRecordsOut
+ */
+ @TestTemplate
+ @DisplayName("Test sink metrics")
+ public void testMetrics(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ int parallelism = 1;
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // make sure use different names when executes multi times
+ String sinkName = "metricTestSink" + testRecords.hashCode();
+ final StreamExecutionEnvironment env =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ env.enableCheckpointing(50);
+
+ DataStreamSource<T> source =
+ env.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "metricTestSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name(sinkName)
+ .setParallelism(parallelism);
+ final JobClient jobClient = env.executeAsync("Metrics Test");
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ try {
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitUntilCondition(
+ () -> {
+ // test metrics
+ try {
+ return compareSinkMetrics(
+ queryRestClient,
+ testEnv,
+ externalContext,
+ jobClient.getJobID(),
+ sinkName,
+ testRecords.size());
+ } catch (Exception e) {
+ // skip failed assert try
+ return false;
+ }
+ },
+ Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
+ } finally {
+ // Clean up
+ killJob(jobClient);
+ }
+ }
+
+ // ----------------------------- Helper Functions ---------------------------------
+
+ /**
+ * Generate a set of test records.
+ *
+ * @param testingSinkSettings sink settings
+ * @param externalContext External context
+ * @return Collection of generated test records
+ */
+ protected List<T> generateTestData(
+ TestingSinkSettings testingSinkSettings,
+ DataStreamSinkExternalContext<T> externalContext) {
+ return externalContext.generateTestData(
+ testingSinkSettings, ThreadLocalRandom.current().nextLong());
+ }
+
+ /**
+ * Poll records from the sink.
+ *
+ * @param result Append records to which list
+ * @param reader The sink reader
+ * @param expected The expected list which help to stop polling
+ * @param retryTimes The retry times
+ * @param semantic The semantic
+ * @return Collection of records in the Sink
+ */
+ private List<T> appendResultData(
+ List<T> result,
+ ExternalSystemDataReader<T> reader,
+ List<T> expected,
+ int retryTimes,
+ CheckpointingMode semantic) {
+ long timeoutMs = 1000L;
+ int retryIndex = 0;
+
+ while (retryIndex++ < retryTimes
+ && !checkGetEnoughRecordsWithSemantic(expected, result, semantic)) {
+ result.addAll(reader.poll(Duration.ofMillis(timeoutMs)));
+ }
+ return result;
+ }
+
+ /**
+ * Check whether the polling should stop.
+ *
+ * @param expected The expected list which help to stop polling
+ * @param result The records that have been read
+ * @param semantic The semantic
+ * @return Whether the polling should stop
+ */
+ private boolean checkGetEnoughRecordsWithSemantic(
+ List<T> expected, List<T> result, CheckpointingMode semantic) {
+ checkNotNull(expected);
+ checkNotNull(result);
+ if (EXACTLY_ONCE.equals(semantic)) {
+ return expected.size() <= result.size();
+ } else if (AT_LEAST_ONCE.equals(semantic)) {
+ Set<Integer> matchedIndex = new HashSet<>();
+ for (T record : expected) {
+ int before = matchedIndex.size();
+ for (int i = 0; i < result.size(); i++) {
+ if (matchedIndex.contains(i)) {
+ continue;
+ }
+ if (record.equals(result.get(i))) {
+ matchedIndex.add(i);
+ break;
+ }
+ }
+ // if not find the record in the result
+ if (before == matchedIndex.size()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ throw new IllegalStateException(
+ String.format("%s delivery guarantee doesn't support test.", semantic.name()));
+ }
+
+ /**
+ * Compare the test data with the result.
Review comment:
```suggestion
* Compare the test data with actual data in given semantic.
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
Review comment:
```suggestion
throw new RuntimeException(String.format("Cannot delete unknown Kafka topic '%s'", topicName), e);
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContextFactory.java
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.connector.testframe.external.ExternalContextFactory;
+
+import org.testcontainers.containers.KafkaContainer;
+
+import java.net.URL;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Kafka table sink external context factory. */
Review comment:
typo: Kafka table ?
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/KafkaSinkITCase.java
##########
@@ -161,6 +177,49 @@ public void tearDown() throws ExecutionException, InterruptedException, TimeoutE
deleteTestTopic(topic);
}
+ /** Integration test based on connector testing framework. */
+ @Nested
+ class IntegrationTests extends SinkTestSuiteBase<String> {
+ // Defines test environment on Flink MiniCluster
+ @SuppressWarnings("unused")
+ @TestEnv
+ MiniClusterTestEnvironment flink = new MiniClusterTestEnvironment();
+
+ // Defines external system
+ @TestExternalSystem
+ DefaultContainerizedExternalSystem<KafkaContainer> kafka =
+ DefaultContainerizedExternalSystem.builder()
+ .fromContainer(
+ new KafkaContainer(
+ DockerImageName.parse(DockerImageVersions.KAFKA)))
+ .build();
+
+ @SuppressWarnings("unused")
+ @TestSemantics
+ CheckpointingMode[] semantics =
+ new CheckpointingMode[] {
+ CheckpointingMode.EXACTLY_ONCE, CheckpointingMode.AT_LEAST_ONCE
+ };
+
+ @SuppressWarnings("unused")
+ @TestContext
+ KafkaSinkExternalContextFactory sinkContext =
+ new KafkaSinkExternalContextFactory(kafka.getContainer(), Collections.emptyList());
+
+ /**
+ * Disable the metric test because of the metric
+ * bug(https://issues.apache.org/jira/browse/FLINK-26126).
+ */
+ @Disabled
Review comment:
```suggestion
@Disabled("Skip metric test until FLINK-26126 fixed")
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
Review comment:
Do we need this methods to be `protected` ?
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
+ config.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ return AdminClient.create(config);
+ }
+
+ @Override
+ public Sink<String> createSink(TestingSinkSettings sinkSettings) {
+ if (!topicExists(topicName)) {
+ createTopic(topicName, 4, (short) 1);
+ }
+
+ KafkaSinkBuilder<String> builder = KafkaSink.builder();
+ Properties properties = new Properties();
+ properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 900000);
+ builder.setBootstrapServers(bootstrapServers)
+ .setDeliverGuarantee(toDeliveryGuarantee(sinkSettings.getCheckpointingMode()))
+ .setTransactionalIdPrefix("testingFramework")
+ .setKafkaProducerConfig(properties)
+ .setRecordSerializer(
+ KafkaRecordSerializationSchema.builder()
+ .setTopic(topicName)
+ .setValueSerializationSchema(new SimpleStringSchema())
+ .build());
+ return builder.build();
+ }
+
+ @Override
+ public ExternalSystemDataReader<String> createSinkDataReader(TestingSinkSettings sinkSettings) {
+ LOG.info("Fetching descriptions for topic: {}", topicName);
Review comment:
I didn't get this meaning of log.
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
+ config.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ return AdminClient.create(config);
+ }
+
+ @Override
+ public Sink<String> createSink(TestingSinkSettings sinkSettings) {
+ if (!topicExists(topicName)) {
+ createTopic(topicName, 4, (short) 1);
+ }
+
+ KafkaSinkBuilder<String> builder = KafkaSink.builder();
+ Properties properties = new Properties();
+ properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 900000);
+ builder.setBootstrapServers(bootstrapServers)
+ .setDeliverGuarantee(toDeliveryGuarantee(sinkSettings.getCheckpointingMode()))
+ .setTransactionalIdPrefix("testingFramework")
+ .setKafkaProducerConfig(properties)
+ .setRecordSerializer(
+ KafkaRecordSerializationSchema.builder()
+ .setTopic(topicName)
+ .setValueSerializationSchema(new SimpleStringSchema())
+ .build());
+ return builder.build();
+ }
+
+ @Override
+ public ExternalSystemDataReader<String> createSinkDataReader(TestingSinkSettings sinkSettings) {
+ LOG.info("Fetching descriptions for topic: {}", topicName);
+ final Map<String, TopicDescription> topicMetadata =
+ getTopicMetadata(Arrays.asList(topicName));
+
+ Set<TopicPartition> subscribedPartitions = new HashSet<>();
+ for (TopicDescription topic : topicMetadata.values()) {
+ for (TopicPartitionInfo partition : topic.partitions()) {
+ subscribedPartitions.add(new TopicPartition(topic.name(), partition.partition()));
+ }
+ }
+
+ Properties properties = new Properties();
+ properties.setProperty(
+ ConsumerConfig.GROUP_ID_CONFIG,
+ "flink-kafka-test" + subscribedPartitions.hashCode());
+ properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.setProperty(
+ ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ properties.setProperty(
+ ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ if (EXACTLY_ONCE.equals(sinkSettings.getCheckpointingMode())) {
+ // default is read_uncommitted
+ properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
+ }
+ properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ readers.add(new KafkaDataReader(properties, subscribedPartitions));
+ return readers.get(readers.size() - 1);
+ }
+
+ @Override
+ public List<String> generateTestData(TestingSinkSettings sinkSettings, long seed) {
+ Random random = new Random(seed);
+ List<String> randomStringRecords = new ArrayList<>();
+ int recordNum =
+ random.nextInt(NUM_RECORDS_UPPER_BOUND - NUM_RECORDS_LOWER_BOUND)
+ + NUM_RECORDS_LOWER_BOUND;
+ for (int i = 0; i < recordNum; i++) {
+ int stringLength = random.nextInt(50) + 1;
+ randomStringRecords.add(generateRandomString(stringLength, random));
+ }
+ return randomStringRecords;
+ }
+
+ private String generateRandomString(int length, Random random) {
+ String alphaNumericString =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; ++i) {
+ sb.append(alphaNumericString.charAt(random.nextInt(alphaNumericString.length())));
+ }
+ return sb.toString();
+ }
+
+ protected Map<String, TopicDescription> getTopicMetadata(List<String> topics) {
+ try {
+ return kafkaAdminClient.describeTopics(topics).all().get();
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format("Failed to get metadata for topics %s.", topics), e);
+ }
+ }
+
+ private boolean topicExists(String topic) {
+ try {
+ kafkaAdminClient.describeTopics(Arrays.asList(topic)).all().get();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (numSplits != 0) {
+ deleteTopic(topicName);
+ }
+ readers.forEach(
Review comment:
check null before cleanup/release resources
##########
File path: flink-end-to-end-tests/flink-end-to-end-tests-common-kafka/src/test/java/org/apache/flink/tests/util/kafka/KafkaSinkE2ECase.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.util.kafka;
+
+import org.apache.flink.connector.kafka.sink.testutils.KafkaSinkExternalContextFactory;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.external.DefaultContainerizedExternalSystem;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.junit.annotations.TestContext;
+import org.apache.flink.connector.testframe.junit.annotations.TestEnv;
+import org.apache.flink.connector.testframe.junit.annotations.TestExternalSystem;
+import org.apache.flink.connector.testframe.junit.annotations.TestSemantics;
+import org.apache.flink.connector.testframe.testsuites.SinkTestSuiteBase;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.tests.util.TestUtils;
+import org.apache.flink.tests.util.flink.FlinkContainerTestEnvironment;
+import org.apache.flink.util.DockerImageVersions;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.util.Arrays;
+
+/** Kafka sink E2E test based on connector testing framework. */
+@SuppressWarnings("unused")
+public class KafkaSinkE2ECase extends SinkTestSuiteBase<String> {
+ private static final String KAFKA_HOSTNAME = "kafka";
+
+ @TestSemantics
+ CheckpointingMode[] semantics =
+ new CheckpointingMode[] {
+ CheckpointingMode.EXACTLY_ONCE, CheckpointingMode.AT_LEAST_ONCE
+ };
+
+ // Defines TestEnvironment
+ @TestEnv FlinkContainerTestEnvironment flink = new FlinkContainerTestEnvironment(1, 6);
+
+ // Defines ConnectorExternalSystem
+ @TestExternalSystem
+ DefaultContainerizedExternalSystem<KafkaContainer> kafka =
+ DefaultContainerizedExternalSystem.builder()
+ .fromContainer(
+ new KafkaContainer(DockerImageName.parse(DockerImageVersions.KAFKA))
+ .withNetworkAliases(KAFKA_HOSTNAME))
+ .bindWithFlinkContainer(flink.getFlinkContainers().getJobManager())
+ .build();
+
+ // Defines 2 External context Factories, so test cases will be invoked twice using these two
+ // kinds of external contexts.
+ @SuppressWarnings("unused")
Review comment:
redundant annotation with line 43
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/external/sink/DataStreamSinkV1ExternalContext.java
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.external.sink;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.connector.sink.Sink;
+
+/**
+ * External context for DataStream sinks which is sink version 1.
Review comment:
```suggestion
* External context for DataStream sinks whose version is V1.
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/external/sink/DataStreamSinkV2ExternalContext.java
##########
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.external.sink;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.connector.sink2.Sink;
+
+/**
+ * External context for DataStream sinks which is sink version 2.
Review comment:
```suggestion
* External context for DataStream sinks whose version is V2.
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
+ config.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ return AdminClient.create(config);
+ }
+
+ @Override
+ public Sink<String> createSink(TestingSinkSettings sinkSettings) {
+ if (!topicExists(topicName)) {
+ createTopic(topicName, 4, (short) 1);
+ }
+
+ KafkaSinkBuilder<String> builder = KafkaSink.builder();
+ Properties properties = new Properties();
+ properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 900000);
+ builder.setBootstrapServers(bootstrapServers)
+ .setDeliverGuarantee(toDeliveryGuarantee(sinkSettings.getCheckpointingMode()))
+ .setTransactionalIdPrefix("testingFramework")
+ .setKafkaProducerConfig(properties)
+ .setRecordSerializer(
+ KafkaRecordSerializationSchema.builder()
+ .setTopic(topicName)
+ .setValueSerializationSchema(new SimpleStringSchema())
+ .build());
+ return builder.build();
+ }
+
+ @Override
+ public ExternalSystemDataReader<String> createSinkDataReader(TestingSinkSettings sinkSettings) {
+ LOG.info("Fetching descriptions for topic: {}", topicName);
+ final Map<String, TopicDescription> topicMetadata =
+ getTopicMetadata(Arrays.asList(topicName));
+
+ Set<TopicPartition> subscribedPartitions = new HashSet<>();
+ for (TopicDescription topic : topicMetadata.values()) {
+ for (TopicPartitionInfo partition : topic.partitions()) {
+ subscribedPartitions.add(new TopicPartition(topic.name(), partition.partition()));
+ }
+ }
+
+ Properties properties = new Properties();
+ properties.setProperty(
+ ConsumerConfig.GROUP_ID_CONFIG,
+ "flink-kafka-test" + subscribedPartitions.hashCode());
+ properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.setProperty(
+ ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ properties.setProperty(
+ ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ if (EXACTLY_ONCE.equals(sinkSettings.getCheckpointingMode())) {
+ // default is read_uncommitted
+ properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
+ }
+ properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ readers.add(new KafkaDataReader(properties, subscribedPartitions));
+ return readers.get(readers.size() - 1);
+ }
+
+ @Override
+ public List<String> generateTestData(TestingSinkSettings sinkSettings, long seed) {
+ Random random = new Random(seed);
+ List<String> randomStringRecords = new ArrayList<>();
+ int recordNum =
+ random.nextInt(NUM_RECORDS_UPPER_BOUND - NUM_RECORDS_LOWER_BOUND)
+ + NUM_RECORDS_LOWER_BOUND;
+ for (int i = 0; i < recordNum; i++) {
+ int stringLength = random.nextInt(50) + 1;
+ randomStringRecords.add(generateRandomString(stringLength, random));
+ }
+ return randomStringRecords;
+ }
+
+ private String generateRandomString(int length, Random random) {
+ String alphaNumericString =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; ++i) {
+ sb.append(alphaNumericString.charAt(random.nextInt(alphaNumericString.length())));
+ }
+ return sb.toString();
Review comment:
The test data pattern `alphaNumericString` and magic number `50` can be constants with one line note
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSource.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumerator;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.util.List;
+
+/**
+ * The source reads data from a list and stops reading at the fixed position. The source will wait
+ * until the checkpoint or savepoint triggers.
+ *
+ * <p>Note that this source must be of parallelism 1.
+ */
+public class FromElementsSource<OUT> implements Source<OUT, FromElementsSplit, NoOpEnumState> {
+ // Boundedness
Review comment:
useless java doc
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
Review comment:
`as source splits.` ?
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSource.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumerator;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.util.List;
+
+/**
+ * The source reads data from a list and stops reading at the fixed position. The source will wait
+ * until the checkpoint or savepoint triggers.
+ *
+ * <p>Note that this source must be of parallelism 1.
Review comment:
```suggestion
* <p>Note: This parallelism of source must be 1.
```
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContextFactory.java
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.connector.testframe.external.ExternalContextFactory;
+
+import org.testcontainers.containers.KafkaContainer;
+
+import java.net.URL;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/** Kafka table sink external context factory. */
+public class KafkaSinkExternalContextFactory
+ implements ExternalContextFactory<KafkaSinkExternalContext> {
+
+ private final KafkaContainer kafkaContainer;
+ private final List<URL> connectorJars;
+
+ public KafkaSinkExternalContextFactory(KafkaContainer kafkaContainer, List<URL> connectorJars) {
+ this.kafkaContainer = kafkaContainer;
+ this.connectorJars = connectorJars;
+ }
+
+ protected String getBootstrapServer() {
Review comment:
No need `protected`
##########
File path: flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/sink/testutils/KafkaSinkExternalContext.java
##########
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.kafka.sink.testutils;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
+import org.apache.flink.connector.kafka.sink.KafkaSink;
+import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.streaming.api.CheckpointingMode;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.TopicPartitionInfo;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+
+/**
+ * A Kafka external context that will create only one topic and use partitions in that topic as
+ * source splits.
+ */
+public class KafkaSinkExternalContext implements DataStreamSinkV2ExternalContext<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSinkExternalContext.class);
+
+ private static final String TOPIC_NAME_PREFIX = "kafka-single-topic";
+ private static final int DEFAULT_TIMEOUT = 30;
+ private static final int NUM_RECORDS_UPPER_BOUND = 500;
+ private static final int NUM_RECORDS_LOWER_BOUND = 100;
+
+ protected String bootstrapServers;
+ protected final String topicName;
+
+ private final List<ExternalSystemDataReader<String>> readers = new ArrayList<>();
+
+ protected int numSplits = 0;
+
+ private List<URL> connectorJarPaths;
+
+ protected final AdminClient kafkaAdminClient;
+
+ public KafkaSinkExternalContext(String bootstrapServers, List<URL> connectorJarPaths) {
+ this.bootstrapServers = bootstrapServers;
+ this.connectorJarPaths = connectorJarPaths;
+ this.topicName =
+ TOPIC_NAME_PREFIX + "-" + ThreadLocalRandom.current().nextLong(Long.MAX_VALUE);
+ kafkaAdminClient = createAdminClient();
+ }
+
+ protected void createTopic(String topicName, int numPartitions, short replicationFactor) {
+ LOG.debug(
+ "Creating new Kafka topic {} with {} partitions and {} replicas",
+ topicName,
+ numPartitions,
+ replicationFactor);
+ NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
+ try {
+ kafkaAdminClient
+ .createTopics(Collections.singletonList(newTopic))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot create topic '%s'", topicName), e);
+ }
+ }
+
+ protected void deleteTopic(String topicName) {
+ LOG.debug("Deleting Kafka topic {}", topicName);
+ try {
+ kafkaAdminClient
+ .deleteTopics(Collections.singletonList(topicName))
+ .all()
+ .get(DEFAULT_TIMEOUT, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ if (ExceptionUtils.getRootCause(e) instanceof UnknownTopicOrPartitionException) {
+ throw new RuntimeException(String.format("Cannot delete topic '%s'", topicName), e);
+ }
+ }
+ }
+
+ private AdminClient createAdminClient() {
+ Properties config = new Properties();
+ config.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ return AdminClient.create(config);
+ }
+
+ @Override
+ public Sink<String> createSink(TestingSinkSettings sinkSettings) {
+ if (!topicExists(topicName)) {
+ createTopic(topicName, 4, (short) 1);
+ }
+
+ KafkaSinkBuilder<String> builder = KafkaSink.builder();
+ Properties properties = new Properties();
+ properties.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 900000);
+ builder.setBootstrapServers(bootstrapServers)
+ .setDeliverGuarantee(toDeliveryGuarantee(sinkSettings.getCheckpointingMode()))
+ .setTransactionalIdPrefix("testingFramework")
+ .setKafkaProducerConfig(properties)
+ .setRecordSerializer(
+ KafkaRecordSerializationSchema.builder()
+ .setTopic(topicName)
+ .setValueSerializationSchema(new SimpleStringSchema())
+ .build());
+ return builder.build();
+ }
+
+ @Override
+ public ExternalSystemDataReader<String> createSinkDataReader(TestingSinkSettings sinkSettings) {
+ LOG.info("Fetching descriptions for topic: {}", topicName);
+ final Map<String, TopicDescription> topicMetadata =
+ getTopicMetadata(Arrays.asList(topicName));
+
+ Set<TopicPartition> subscribedPartitions = new HashSet<>();
+ for (TopicDescription topic : topicMetadata.values()) {
+ for (TopicPartitionInfo partition : topic.partitions()) {
+ subscribedPartitions.add(new TopicPartition(topic.name(), partition.partition()));
+ }
+ }
+
+ Properties properties = new Properties();
+ properties.setProperty(
+ ConsumerConfig.GROUP_ID_CONFIG,
+ "flink-kafka-test" + subscribedPartitions.hashCode());
+ properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ properties.setProperty(
+ ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ properties.setProperty(
+ ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
+ StringDeserializer.class.getCanonicalName());
+ if (EXACTLY_ONCE.equals(sinkSettings.getCheckpointingMode())) {
+ // default is read_uncommitted
+ properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed");
+ }
+ properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ readers.add(new KafkaDataReader(properties, subscribedPartitions));
+ return readers.get(readers.size() - 1);
+ }
+
+ @Override
+ public List<String> generateTestData(TestingSinkSettings sinkSettings, long seed) {
+ Random random = new Random(seed);
+ List<String> randomStringRecords = new ArrayList<>();
+ int recordNum =
+ random.nextInt(NUM_RECORDS_UPPER_BOUND - NUM_RECORDS_LOWER_BOUND)
+ + NUM_RECORDS_LOWER_BOUND;
+ for (int i = 0; i < recordNum; i++) {
+ int stringLength = random.nextInt(50) + 1;
+ randomStringRecords.add(generateRandomString(stringLength, random));
+ }
+ return randomStringRecords;
+ }
+
+ private String generateRandomString(int length, Random random) {
+ String alphaNumericString =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789";
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; ++i) {
+ sb.append(alphaNumericString.charAt(random.nextInt(alphaNumericString.length())));
+ }
+ return sb.toString();
+ }
+
+ protected Map<String, TopicDescription> getTopicMetadata(List<String> topics) {
+ try {
+ return kafkaAdminClient.describeTopics(topics).all().get();
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format("Failed to get metadata for topics %s.", topics), e);
+ }
+ }
+
+ private boolean topicExists(String topic) {
+ try {
+ kafkaAdminClient.describeTopics(Arrays.asList(topic)).all().get();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (numSplits != 0) {
+ deleteTopic(topicName);
+ }
+ readers.forEach(
+ reader -> {
+ try {
+ reader.close();
+ } catch (Exception e) {
+ kafkaAdminClient.close();
+ throw new RuntimeException("Cannot close split writer", e);
+ }
+ });
+ readers.clear();
+ kafkaAdminClient.close();
+ }
+
+ @Override
+ public String toString() {
+ return "Single-topic Kafka";
+ }
+
+ @Override
+ public List<URL> getConnectorJarPaths() {
+ return connectorJarPaths;
+ }
+
+ @Override
+ public TypeInformation<String> getProducedType() {
+ return TypeInformation.of(String.class);
+ }
+
+ private DeliveryGuarantee toDeliveryGuarantee(CheckpointingMode checkpointingMode) {
+ switch (checkpointingMode) {
+ case EXACTLY_ONCE:
+ return DeliveryGuarantee.EXACTLY_ONCE;
+ case AT_LEAST_ONCE:
+ return DeliveryGuarantee.AT_LEAST_ONCE;
+ default:
+ throw new IllegalArgumentException(
+ "Only exactly-once and al-least-once checkpointing mode are supported");
Review comment:
```suggestion
throw new IllegalArgumentException(
String.format("Only exactly-once and al-least-once checkpointing mode are supported, but actual is %s.", checkpointingMode));
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSource.java
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumState;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumStateSerializer;
+import org.apache.flink.connector.testframe.source.enumerator.NoOpEnumerator;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.util.List;
+
+/**
+ * The source reads data from a list and stops reading at the fixed position. The source will wait
+ * until the checkpoint or savepoint triggers.
+ *
+ * <p>Note that this source must be of parallelism 1.
+ */
+public class FromElementsSource<OUT> implements Source<OUT, FromElementsSplit, NoOpEnumState> {
+ // Boundedness
+ private Boundedness boundedness;
+
+ private List<OUT> elements;
+
+ private Integer successNum;
+
+ public FromElementsSource(List<OUT> elements) {
+ this.elements = elements;
+ }
+
+ public FromElementsSource(Boundedness boundedness, List<OUT> elements, Integer successNum) {
Review comment:
how about `emittedElementsNum` ?
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSourceReader.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.metrics.Counter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE;
+
+/** The source reader for collections of elements. */
+public class FromElementsSourceReader<T> implements SourceReader<T, FromElementsSplit> {
+ private static final Logger LOG = LoggerFactory.getLogger(FromElementsSourceReader.class);
+
+ protected volatile int numElementsEmitted;
+ protected volatile boolean isRunning = true;
+
+ /** The context of this source reader. */
+ protected SourceReaderContext context;
+
+ protected List<T> elements;
+ protected Counter numRecordInCounter;
+
+ public FromElementsSourceReader(List<T> elements, SourceReaderContext context) {
+ this.context = context;
+ this.numElementsEmitted = 0;
+ this.elements = elements;
+ this.numRecordInCounter = context.metricGroup().getIOMetricGroup().getNumRecordsInCounter();
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<T> output) throws Exception {
+ if (isRunning && numElementsEmitted < elements.size()) {
+ output.collect(elements.get(numElementsEmitted));
+ numElementsEmitted++;
+ numRecordInCounter.inc();
+ return MORE_AVAILABLE;
+ }
+ return InputStatus.END_OF_INPUT;
+ }
+
+ @Override
+ public List<FromElementsSplit> snapshotState(long checkpointId) {
+ return Arrays.asList(new FromElementsSplit(numElementsEmitted));
+ }
+
+ @Override
+ public CompletableFuture<Void> isAvailable() {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ future.complete(null);
+ return future;
Review comment:
return CompletableFuture.completedFuture(null);
##########
File path: flink-end-to-end-tests/flink-end-to-end-tests-common-kafka/src/test/java/org/apache/flink/tests/util/kafka/KafkaSinkE2ECase.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.util.kafka;
+
+import org.apache.flink.connector.kafka.sink.testutils.KafkaSinkExternalContextFactory;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.external.DefaultContainerizedExternalSystem;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.junit.annotations.TestContext;
+import org.apache.flink.connector.testframe.junit.annotations.TestEnv;
+import org.apache.flink.connector.testframe.junit.annotations.TestExternalSystem;
+import org.apache.flink.connector.testframe.junit.annotations.TestSemantics;
+import org.apache.flink.connector.testframe.testsuites.SinkTestSuiteBase;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.tests.util.TestUtils;
+import org.apache.flink.tests.util.flink.FlinkContainerTestEnvironment;
+import org.apache.flink.util.DockerImageVersions;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.util.Arrays;
+
+/** Kafka sink E2E test based on connector testing framework. */
+@SuppressWarnings("unused")
+public class KafkaSinkE2ECase extends SinkTestSuiteBase<String> {
+ private static final String KAFKA_HOSTNAME = "kafka";
+
+ @TestSemantics
+ CheckpointingMode[] semantics =
+ new CheckpointingMode[] {
+ CheckpointingMode.EXACTLY_ONCE, CheckpointingMode.AT_LEAST_ONCE
+ };
+
+ // Defines TestEnvironment
+ @TestEnv FlinkContainerTestEnvironment flink = new FlinkContainerTestEnvironment(1, 6);
+
+ // Defines ConnectorExternalSystem
+ @TestExternalSystem
+ DefaultContainerizedExternalSystem<KafkaContainer> kafka =
+ DefaultContainerizedExternalSystem.builder()
+ .fromContainer(
+ new KafkaContainer(DockerImageName.parse(DockerImageVersions.KAFKA))
+ .withNetworkAliases(KAFKA_HOSTNAME))
+ .bindWithFlinkContainer(flink.getFlinkContainers().getJobManager())
+ .build();
+
+ // Defines 2 External context Factories, so test cases will be invoked twice using these two
+ // kinds of external contexts.
+ @SuppressWarnings("unused")
+ @TestContext
+ KafkaSinkExternalContextFactory contextFactory =
+ new KafkaSinkExternalContextFactory(
+ kafka.getContainer(),
+ Arrays.asList(
+ TestUtils.getResource("kafka-connector.jar")
+ .toAbsolutePath()
+ .toUri()
+ .toURL(),
+ TestUtils.getResource("kafka-clients.jar")
+ .toAbsolutePath()
+ .toUri()
+ .toURL(),
+ TestUtils.getResource("flink-connector-testing.jar")
+ .toAbsolutePath()
+ .toUri()
+ .toURL()));
+
+ public KafkaSinkE2ECase() throws Exception {}
+
+ /**
+ * Disable the metric test because of the metric
+ * bug(https://issues.apache.org/jira/browse/FLINK-26126).
+ */
+ @Disabled
Review comment:
```suggestion
@Disabled("Skip metric test until FLINK-26126 fixed")
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSourceReader.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.metrics.Counter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE;
+
+/** The source reader for collections of elements. */
+public class FromElementsSourceReader<T> implements SourceReader<T, FromElementsSplit> {
+ private static final Logger LOG = LoggerFactory.getLogger(FromElementsSourceReader.class);
+
+ protected volatile int numElementsEmitted;
+ protected volatile boolean isRunning = true;
+
+ /** The context of this source reader. */
+ protected SourceReaderContext context;
+
+ protected List<T> elements;
+ protected Counter numRecordInCounter;
+
+ public FromElementsSourceReader(List<T> elements, SourceReaderContext context) {
+ this.context = context;
+ this.numElementsEmitted = 0;
+ this.elements = elements;
+ this.numRecordInCounter = context.metricGroup().getIOMetricGroup().getNumRecordsInCounter();
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<T> output) throws Exception {
+ if (isRunning && numElementsEmitted < elements.size()) {
+ output.collect(elements.get(numElementsEmitted));
+ numElementsEmitted++;
+ numRecordInCounter.inc();
+ return MORE_AVAILABLE;
+ }
+ return InputStatus.END_OF_INPUT;
+ }
+
+ @Override
+ public List<FromElementsSplit> snapshotState(long checkpointId) {
+ return Arrays.asList(new FromElementsSplit(numElementsEmitted));
+ }
+
+ @Override
+ public CompletableFuture<Void> isAvailable() {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ future.complete(null);
+ return future;
+ }
+
+ @Override
+ public void addSplits(List<FromElementsSplit> splits) {
+ numElementsEmitted = splits.get(0).getEmitNum();
+ LOG.info("ListSourceReader restores from {}.", numElementsEmitted);
+ }
+
+ @Override
+ public void notifyNoMoreSplits() {}
+
+ @Override
+ public void close() throws Exception {
+ isRunning = false;
+ }
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) throws Exception {
+ LOG.info("{} checkpoint finished.", checkpointId);
Review comment:
minor:
```suggestion
LOG.info("checkpoint {} finished.", checkpointId);
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSourceReader.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.metrics.Counter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE;
+
+/** The source reader for collections of elements. */
+public class FromElementsSourceReader<T> implements SourceReader<T, FromElementsSplit> {
+ private static final Logger LOG = LoggerFactory.getLogger(FromElementsSourceReader.class);
+
+ protected volatile int numElementsEmitted;
+ protected volatile boolean isRunning = true;
+
+ /** The context of this source reader. */
+ protected SourceReaderContext context;
+
+ protected List<T> elements;
+ protected Counter numRecordInCounter;
+
+ public FromElementsSourceReader(List<T> elements, SourceReaderContext context) {
+ this.context = context;
+ this.numElementsEmitted = 0;
+ this.elements = elements;
+ this.numRecordInCounter = context.metricGroup().getIOMetricGroup().getNumRecordsInCounter();
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<T> output) throws Exception {
+ if (isRunning && numElementsEmitted < elements.size()) {
+ output.collect(elements.get(numElementsEmitted));
+ numElementsEmitted++;
+ numRecordInCounter.inc();
+ return MORE_AVAILABLE;
+ }
+ return InputStatus.END_OF_INPUT;
+ }
+
+ @Override
+ public List<FromElementsSplit> snapshotState(long checkpointId) {
+ return Arrays.asList(new FromElementsSplit(numElementsEmitted));
+ }
+
+ @Override
+ public CompletableFuture<Void> isAvailable() {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ future.complete(null);
+ return future;
+ }
+
+ @Override
+ public void addSplits(List<FromElementsSplit> splits) {
+ numElementsEmitted = splits.get(0).getEmitNum();
+ LOG.info("ListSourceReader restores from {}.", numElementsEmitted);
Review comment:
```suggestion
LOG.info("FromElementsSourceReader restores from {}.", numElementsEmitted);
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/split/FromElementsSplitSerializer.java
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source.split;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/** The split serializer for the list source. */
Review comment:
```suggestion
/** The split serializer for the {@link FromElementsSource}. */
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSourceReader.java
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.metrics.Counter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE;
+
+/** The source reader for collections of elements. */
+public class FromElementsSourceReader<T> implements SourceReader<T, FromElementsSplit> {
+ private static final Logger LOG = LoggerFactory.getLogger(FromElementsSourceReader.class);
+
+ protected volatile int numElementsEmitted;
+ protected volatile boolean isRunning = true;
+
+ /** The context of this source reader. */
+ protected SourceReaderContext context;
+
+ protected List<T> elements;
+ protected Counter numRecordInCounter;
+
+ public FromElementsSourceReader(List<T> elements, SourceReaderContext context) {
+ this.context = context;
+ this.numElementsEmitted = 0;
+ this.elements = elements;
+ this.numRecordInCounter = context.metricGroup().getIOMetricGroup().getNumRecordsInCounter();
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<T> output) throws Exception {
+ if (isRunning && numElementsEmitted < elements.size()) {
Review comment:
`emittedNum` ?
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/source/FromElementsSourceReaderWithSuccessNum.java
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.source;
+
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.connector.testframe.source.split.FromElementsSplit;
+import org.apache.flink.core.io.InputStatus;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.core.io.InputStatus.MORE_AVAILABLE;
+
+/** The reader reads data from a list. */
Review comment:
Add more note to explain the diff with `FromElementsSourceReader`, how about the name `FromLimitedElementsSourceReader` and `int limitedNum` ?
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
Review comment:
```suggestion
* Test connector sink restart from a completed savepoint with a higher parallelism.
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
Review comment:
```suggestion
/**
* Test DataStream connector sink.
*
* <p>The following tests will create a sink in the external system, generate a collection of test data
* and write them to this sink by the Flink Job.
*
* <p>In order to pass these tests, the number of records produced by Flink need to be equals to
* the generated test data. And the records in the sink will be compared to the test data by the
* different semantics. There's no requirement for records order.
*/
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a higher parallelism 4.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a higher parallelism")
+ public void testScaleUp(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 4);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a lower parallelism.
Review comment:
```suggestion
* Test connector sink restart from a completed savepoint with a lower parallelism.
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/external/sink/DataStreamSinkV1ExternalContext.java
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.external.sink;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.connector.sink.Sink;
+
+/**
+ * External context for DataStream sinks which is sink version 1.
Review comment:
BTW, do we have any test for v1 sink.
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a higher parallelism 4.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a higher parallelism")
+ public void testScaleUp(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 4);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a lower parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 4 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a lower parallelism 2.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a lower parallelism")
+ public void testScaleDown(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 4, 2);
+ }
+
+ private void restartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic,
+ final int beforeParallelism,
+ final int afterParallelism)
+ throws Exception {
+ // Step 1: Preparation
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.setRestartStrategy(RestartStrategies.noRestart());
+
+ // Step 2: Generate test data
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Step 3: Build and execute Flink job
+ int numBeforeSuccess = testRecords.size() / 2;
+ DataStreamSource<T> source =
+ execEnv.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ numBeforeSuccess),
+ WatermarkStrategy.noWatermarks(),
+ "beforeRestartSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name("Sink restart test")
+ .setParallelism(beforeParallelism);
+
+ /**
+ * The job should stop after consume a specified number of records. In order to know when
+ * the specified number of records have been consumed, a collect sink is need to be watched.
+ */
+ CollectResultIterator<T> iterator = addCollectSink(source);
+ final JobClient jobClient = execEnv.executeAsync("Restart Test");
+ iterator.setJobClient(jobClient);
+
+ // Step 4: Wait for the expected result and stop Flink job with a savepoint
+ String savepointDir;
Review comment:
```suggestion
String savepointPath;
```
##########
File path: flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.testframe.testsuites;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.testframe.environment.TestEnvironment;
+import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
+import org.apache.flink.connector.testframe.external.ExternalSystemDataReader;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV1ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.DataStreamSinkV2ExternalContext;
+import org.apache.flink.connector.testframe.external.sink.TestingSinkSettings;
+import org.apache.flink.connector.testframe.junit.extensions.ConnectorTestingExtension;
+import org.apache.flink.connector.testframe.junit.extensions.TestCaseInvocationContextProvider;
+import org.apache.flink.connector.testframe.source.FromElementsSource;
+import org.apache.flink.connector.testframe.utils.CollectIteratorAssertions;
+import org.apache.flink.connector.testframe.utils.MetricQuerier;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.core.execution.SavepointFormatType;
+import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.collect.CollectResultIterator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperator;
+import org.apache.flink.streaming.api.operators.collect.CollectSinkOperatorFactory;
+import org.apache.flink.streaming.api.operators.collect.CollectStreamSink;
+import org.apache.flink.util.TestLoggerExtension;
+
+import org.apache.commons.math3.util.Precision;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.opentest4j.TestAbortedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_COLLECT_DATA_TIMEOUT;
+import static org.apache.flink.connector.testframe.utils.ConnectorTestConstants.DEFAULT_JOB_STATUS_CHANGE_TIMEOUT;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.terminateJob;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForJobStatus;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitUntilCondition;
+import static org.apache.flink.streaming.api.CheckpointingMode.AT_LEAST_ONCE;
+import static org.apache.flink.streaming.api.CheckpointingMode.EXACTLY_ONCE;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Base class for sink test suite.
+ *
+ * <p>All cases should have well-descriptive JavaDoc, including:
+ *
+ * <ul>
+ * <li>What's the purpose of this case
+ * <li>Simple description of how this case works
+ * <li>Condition to fulfill in order to pass this case
+ * <li>Requirement of running this case
+ * </ul>
+ */
+@ExtendWith({
+ ConnectorTestingExtension.class,
+ TestLoggerExtension.class,
+ TestCaseInvocationContextProvider.class
+})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Experimental
+public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(SinkTestSuiteBase.class);
+
+ // ----------------------------- Basic test cases ---------------------------------
+
+ /**
+ * Test connector data stream sink.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write them to this sink by the Flink Job.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test data stream sink")
+ public void testBasicSink(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Build and execute Flink job
+ StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.enableCheckpointing(50);
+ DataStream<T> dataStream =
+ execEnv.fromCollection(testRecords)
+ .name("sourceInSinkTest")
+ .setParallelism(1)
+ .returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .setParallelism(1)
+ .name("sinkInSinkTest");
+ final JobClient jobClient = execEnv.executeAsync("DataStream Sink Test");
+
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ // Check test result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with the same parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint. After the job has been
+ * running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting from a savepoint")
+ public void testStartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 2);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a higher parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 2 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a higher parallelism 4.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a higher parallelism")
+ public void testScaleUp(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 2, 4);
+ }
+
+ /**
+ * Test connector source restart from a completed savepoint with a lower parallelism.
+ *
+ * <p>This test will create a sink in the external system, generate a collection of test data
+ * and write a half part of them to this sink by the Flink Job with parallelism 4 at first. Then
+ * stop the job, restart the same job from the completed savepoint with a lower parallelism 2.
+ * After the job has been running, write the other part to the sink and compare the result.
+ *
+ * <p>In order to pass this test, the number of records produced by Flink need to be equals to
+ * the generated test data. And the records in the sink will be compared to the test data by the
+ * different semantic. There's no requirement for record order.
+ */
+ @TestTemplate
+ @DisplayName("Test sink restarting with a lower parallelism")
+ public void testScaleDown(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ restartFromSavepoint(testEnv, externalContext, semantic, 4, 2);
+ }
+
+ private void restartFromSavepoint(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic,
+ final int beforeParallelism,
+ final int afterParallelism)
+ throws Exception {
+ // Step 1: Preparation
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ final StreamExecutionEnvironment execEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ execEnv.setRestartStrategy(RestartStrategies.noRestart());
+
+ // Step 2: Generate test data
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // Step 3: Build and execute Flink job
+ int numBeforeSuccess = testRecords.size() / 2;
+ DataStreamSource<T> source =
+ execEnv.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ numBeforeSuccess),
+ WatermarkStrategy.noWatermarks(),
+ "beforeRestartSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name("Sink restart test")
+ .setParallelism(beforeParallelism);
+
+ /**
+ * The job should stop after consume a specified number of records. In order to know when
+ * the specified number of records have been consumed, a collect sink is need to be watched.
+ */
+ CollectResultIterator<T> iterator = addCollectSink(source);
+ final JobClient jobClient = execEnv.executeAsync("Restart Test");
+ iterator.setJobClient(jobClient);
+
+ // Step 4: Wait for the expected result and stop Flink job with a savepoint
+ String savepointDir;
+ try {
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitExpectedSizeData(iterator, numBeforeSuccess);
+
+ savepointDir =
+ jobClient
+ .stopWithSavepoint(
+ true, testEnv.getCheckpointUri(), SavepointFormatType.CANONICAL)
+ .get(30, TimeUnit.SECONDS);
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.FINISHED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+ } catch (Exception e) {
+ killJob(jobClient);
+ throw e;
+ }
+
+ List<T> target = testRecords.subList(0, numBeforeSuccess);
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), target, semantic);
+
+ // Step 4: restart the Flink job with the savepoint
+ final StreamExecutionEnvironment restartEnv =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .setSavepointRestorePath(savepointDir)
+ .build());
+ restartEnv.enableCheckpointing(50);
+
+ DataStreamSource<T> restartSource =
+ restartEnv
+ .fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "restartSource")
+ .setParallelism(1);
+
+ DataStream<T> sinkStream = restartSource.returns(externalContext.getProducedType());
+ tryCreateSink(sinkStream, externalContext, sinkSettings).setParallelism(afterParallelism);
+ addCollectSink(restartSource);
+ final JobClient restartJobClient = restartEnv.executeAsync("Restart Test");
+
+ try {
+ // Check the result
+ checkResultWithSemantic(
+ externalContext.createSinkDataReader(sinkSettings), testRecords, semantic);
+ } finally {
+ killJob(restartJobClient);
+ iterator.close();
+ }
+ }
+
+ /**
+ * Test connector sink metrics.
+ *
+ * <p>This test will create a sink in the external system, generate test data and write them to
+ * the sink via a Flink job. Then read and compare the metrics.
+ *
+ * <p>Now test: numRecordsOut
+ */
+ @TestTemplate
+ @DisplayName("Test sink metrics")
+ public void testMetrics(
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> externalContext,
+ CheckpointingMode semantic)
+ throws Exception {
+ TestingSinkSettings sinkSettings = getTestingSinkSettings(semantic);
+ int parallelism = 1;
+ final List<T> testRecords = generateTestData(sinkSettings, externalContext);
+
+ // make sure use different names when executes multi times
+ String sinkName = "metricTestSink" + testRecords.hashCode();
+ final StreamExecutionEnvironment env =
+ testEnv.createExecutionEnvironment(
+ TestEnvironmentSettings.builder()
+ .setConnectorJarPaths(externalContext.getConnectorJarPaths())
+ .build());
+ env.enableCheckpointing(50);
+
+ DataStreamSource<T> source =
+ env.fromSource(
+ new FromElementsSource<>(
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ testRecords,
+ testRecords.size()),
+ WatermarkStrategy.noWatermarks(),
+ "metricTestSource")
+ .setParallelism(1);
+
+ DataStream<T> dataStream = source.returns(externalContext.getProducedType());
+ tryCreateSink(dataStream, externalContext, sinkSettings)
+ .name(sinkName)
+ .setParallelism(parallelism);
+ final JobClient jobClient = env.executeAsync("Metrics Test");
+ final MetricQuerier queryRestClient = new MetricQuerier(new Configuration());
+ try {
+ waitForAllTaskRunning(
+ () ->
+ queryRestClient.getJobDetails(
+ testEnv.getRestEndpoint(), jobClient.getJobID()),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+
+ waitUntilCondition(
+ () -> {
+ // test metrics
+ try {
+ return compareSinkMetrics(
+ queryRestClient,
+ testEnv,
+ externalContext,
+ jobClient.getJobID(),
+ sinkName,
+ testRecords.size());
+ } catch (Exception e) {
+ // skip failed assert try
+ return false;
+ }
+ },
+ Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
+ } finally {
+ // Clean up
+ killJob(jobClient);
+ }
+ }
+
+ // ----------------------------- Helper Functions ---------------------------------
+
+ /**
+ * Generate a set of test records.
+ *
+ * @param testingSinkSettings sink settings
+ * @param externalContext External context
+ * @return Collection of generated test records
+ */
+ protected List<T> generateTestData(
+ TestingSinkSettings testingSinkSettings,
+ DataStreamSinkExternalContext<T> externalContext) {
+ return externalContext.generateTestData(
+ testingSinkSettings, ThreadLocalRandom.current().nextLong());
+ }
+
+ /**
+ * Poll records from the sink.
+ *
+ * @param result Append records to which list
+ * @param reader The sink reader
+ * @param expected The expected list which help to stop polling
+ * @param retryTimes The retry times
+ * @param semantic The semantic
+ * @return Collection of records in the Sink
+ */
+ private List<T> appendResultData(
+ List<T> result,
+ ExternalSystemDataReader<T> reader,
+ List<T> expected,
+ int retryTimes,
+ CheckpointingMode semantic) {
+ long timeoutMs = 1000L;
+ int retryIndex = 0;
+
+ while (retryIndex++ < retryTimes
+ && !checkGetEnoughRecordsWithSemantic(expected, result, semantic)) {
+ result.addAll(reader.poll(Duration.ofMillis(timeoutMs)));
+ }
+ return result;
+ }
+
+ /**
+ * Check whether the polling should stop.
+ *
+ * @param expected The expected list which help to stop polling
+ * @param result The records that have been read
+ * @param semantic The semantic
+ * @return Whether the polling should stop
+ */
+ private boolean checkGetEnoughRecordsWithSemantic(
+ List<T> expected, List<T> result, CheckpointingMode semantic) {
+ checkNotNull(expected);
+ checkNotNull(result);
+ if (EXACTLY_ONCE.equals(semantic)) {
+ return expected.size() <= result.size();
+ } else if (AT_LEAST_ONCE.equals(semantic)) {
+ Set<Integer> matchedIndex = new HashSet<>();
+ for (T record : expected) {
+ int before = matchedIndex.size();
+ for (int i = 0; i < result.size(); i++) {
+ if (matchedIndex.contains(i)) {
+ continue;
+ }
+ if (record.equals(result.get(i))) {
+ matchedIndex.add(i);
+ break;
+ }
+ }
+ // if not find the record in the result
+ if (before == matchedIndex.size()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ throw new IllegalStateException(
+ String.format("%s delivery guarantee doesn't support test.", semantic.name()));
+ }
+
+ /**
+ * Compare the test data with the result.
+ *
+ * @param reader the data reader for the sink
+ * @param testData the test data
+ * @param semantic the supported semantic, see {@link CheckpointingMode}
+ */
+ private void checkResultWithSemantic(
+ ExternalSystemDataReader<T> reader, List<T> testData, CheckpointingMode semantic)
+ throws Exception {
+ final ArrayList<T> result = new ArrayList<>();
+ waitUntilCondition(
+ () -> {
+ appendResultData(result, reader, testData, 30, semantic);
+ try {
+ CollectIteratorAssertions.assertThat(sort(result).iterator())
+ .matchesRecordsFromSource(Arrays.asList(sort(testData)), semantic);
+ return true;
+ } catch (Throwable t) {
+ return false;
+ }
+ },
+ Deadline.fromNow(DEFAULT_COLLECT_DATA_TIMEOUT));
+ }
+
+ /** Compare the metrics. */
+ private boolean compareSinkMetrics(
+ MetricQuerier metricQuerier,
+ TestEnvironment testEnv,
+ DataStreamSinkExternalContext<T> context,
+ JobID jobId,
+ String sinkName,
+ long allRecordSize)
+ throws Exception {
+ double sumNumRecordsOut =
+ metricQuerier.getAggregatedMetricsByRestAPI(
+ testEnv.getRestEndpoint(),
+ jobId,
+ sinkName,
+ MetricNames.IO_NUM_RECORDS_OUT,
+ getSinkMetricFilter(context));
+ return Precision.equals(allRecordSize, sumNumRecordsOut);
+ }
+
+ /** Sort the list. */
+ private List<T> sort(List<T> list) {
+ return list.stream().sorted().collect(Collectors.toList());
+ }
+
+ private TestingSinkSettings getTestingSinkSettings(CheckpointingMode checkpointingMode) {
+ return TestingSinkSettings.builder().setCheckpointingMode(checkpointingMode).build();
+ }
+
+ private void killJob(JobClient jobClient) throws Exception {
+ terminateJob(jobClient);
+ waitForJobStatus(
+ jobClient,
+ Collections.singletonList(JobStatus.CANCELED),
+ Deadline.fromNow(DEFAULT_JOB_STATUS_CHANGE_TIMEOUT));
+ }
+
+ private DataStreamSink<T> tryCreateSink(
+ DataStream<T> dataStream,
+ DataStreamSinkExternalContext<T> context,
+ TestingSinkSettings sinkSettings) {
+ try {
+ if (context instanceof DataStreamSinkV1ExternalContext) {
+ org.apache.flink.api.connector.sink.Sink<T, ?, ?, ?> sinkV1 =
+ ((DataStreamSinkV1ExternalContext<T>) context).createSink(sinkSettings);
+ return dataStream.sinkTo(sinkV1);
+ } else if (context instanceof DataStreamSinkV2ExternalContext) {
+ Sink<T> sinkV2 =
+ ((DataStreamSinkV2ExternalContext<T>) context).createSink(sinkSettings);
+ return dataStream.sinkTo(sinkV2);
+ } else {
+ throw new IllegalArgumentException(
+ String.format("Get unexpected sink context: %s", context.getClass()));
Review comment:
hint, please use the log pattern: The supported are ... , but actual is ..
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org