You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jq...@apache.org on 2021/09/18 13:45:43 UTC
[flink] 02/03: [FLINK-24277][connector/kafka] Remove auto-generated
group id in Kafka table source
This is an automated email from the ASF dual-hosted git repository.
jqin pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 984cfe9797562d578d54329e8660758d777d80ec
Author: Qingsheng Ren <re...@gmail.com>
AuthorDate: Tue Sep 14 15:24:10 2021 +0800
[FLINK-24277][connector/kafka] Remove auto-generated group id in Kafka table source
---
.../connectors/kafka/table/KafkaDynamicSource.java | 16 ----------
.../kafka/source/KafkaSourceTestUtils.java | 6 ++++
.../kafka/table/KafkaDynamicTableFactoryTest.java | 34 +++++++++++++++++++++-
3 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
index 8e12124..ab0fa13 100644
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
+++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
@@ -49,12 +49,9 @@ import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.utils.DataTypeUtils;
import org.apache.flink.util.Preconditions;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.header.Header;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
@@ -77,8 +74,6 @@ import java.util.stream.Stream;
public class KafkaDynamicSource
implements ScanTableSource, SupportsReadingMetadata, SupportsWatermarkPushDown {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaDynamicSource.class);
-
// --------------------------------------------------------------------------------------------
// Mutable attributes
// --------------------------------------------------------------------------------------------
@@ -389,17 +384,6 @@ public class KafkaDynamicSource
kafkaSourceBuilder.setTopicPattern(topicPattern);
}
- // For compatibility with legacy source that is not validating group id
- if (!properties.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
- String generatedGroupId = "KafkaSource-" + tableIdentifier;
- LOG.warn(
- "Property \"{}\" is required for offset commit but not set in table options. "
- + "Assigning \"{}\" as consumer group id",
- ConsumerConfig.GROUP_ID_CONFIG,
- generatedGroupId);
- kafkaSourceBuilder.setGroupId(generatedGroupId);
- }
-
switch (startupMode) {
case EARLIEST:
kafkaSourceBuilder.setStartingOffsets(OffsetsInitializer.earliest());
diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
index fce9591..572b77d 100644
--- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
+++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
@@ -19,6 +19,7 @@
package org.apache.flink.connector.kafka.source;
import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.kafka.source.reader.KafkaSourceReader;
import java.util.Collection;
@@ -44,4 +45,9 @@ public class KafkaSourceTestUtils {
return ((KafkaSourceReader<T>)
kafkaSource.createReader(sourceReaderContext, splitFinishedHook));
}
+
+ /** Get configuration of KafkaSource. */
+ public static Configuration getKafkaSourceConfiguration(KafkaSource<?> kafkaSource) {
+ return kafkaSource.getConfiguration();
+ }
}
diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
index 01af4b0..a0cc3cf 100644
--- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
+++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
@@ -22,9 +22,13 @@ import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.api.connector.sink.Sink;
import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.base.DeliveryGuarantee;
import org.apache.flink.connector.kafka.sink.KafkaSink;
import org.apache.flink.connector.kafka.source.KafkaSource;
+import org.apache.flink.connector.kafka.source.KafkaSourceOptions;
+import org.apache.flink.connector.kafka.source.KafkaSourceTestUtils;
import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumState;
import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
import org.apache.flink.formats.avro.AvroRowDataSerializationSchema;
@@ -68,6 +72,7 @@ import org.apache.flink.util.TestLogger;
import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -93,6 +98,7 @@ import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSou
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -350,6 +356,31 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
}
@Test
+ public void testTableSourceCommitOnCheckpointDisabled() {
+ final Map<String, String> modifiedOptions =
+ getModifiedOptions(
+ getBasicSourceOptions(), options -> options.remove("properties.group.id"));
+ final DynamicTableSource tableSource = createTableSource(SCHEMA, modifiedOptions);
+
+ assertThat(tableSource, instanceOf(KafkaDynamicSource.class));
+ ScanTableSource.ScanRuntimeProvider providerWithoutGroupId =
+ ((KafkaDynamicSource) tableSource)
+ .getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
+ assertThat(providerWithoutGroupId, instanceOf(DataStreamScanProvider.class));
+ final KafkaSource<?> kafkaSource = assertKafkaSource(providerWithoutGroupId);
+ final Configuration configuration =
+ KafkaSourceTestUtils.getKafkaSourceConfiguration(kafkaSource);
+
+ // Test offset commit on checkpoint should be disabled when do not set consumer group.
+ assertFalse(configuration.get(KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT));
+ assertFalse(
+ configuration.get(
+ ConfigOptions.key(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)
+ .booleanType()
+ .noDefaultValue()));
+ }
+
+ @Test
public void testTableSink() {
final Map<String, String> modifiedOptions =
getModifiedOptions(
@@ -997,7 +1028,7 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
return tableOptions;
}
- private void assertKafkaSource(ScanTableSource.ScanRuntimeProvider provider) {
+ private KafkaSource<?> assertKafkaSource(ScanTableSource.ScanRuntimeProvider provider) {
assertThat(provider, instanceOf(DataStreamScanProvider.class));
final DataStreamScanProvider dataStreamScanProvider = (DataStreamScanProvider) provider;
final Transformation<RowData> transformation =
@@ -1010,5 +1041,6 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
(SourceTransformation<RowData, KafkaPartitionSplit, KafkaSourceEnumState>)
transformation;
assertThat(sourceTransformation.getSource(), instanceOf(KafkaSource.class));
+ return (KafkaSource<?>) sourceTransformation.getSource();
}
}