You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jq...@apache.org on 2021/09/18 13:45:43 UTC

[flink] 02/03: [FLINK-24277][connector/kafka] Remove auto-generated group id in Kafka table source

This is an automated email from the ASF dual-hosted git repository.

jqin pushed a commit to branch release-1.14
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 984cfe9797562d578d54329e8660758d777d80ec
Author: Qingsheng Ren <re...@gmail.com>
AuthorDate: Tue Sep 14 15:24:10 2021 +0800

    [FLINK-24277][connector/kafka] Remove auto-generated group id in Kafka table source
---
 .../connectors/kafka/table/KafkaDynamicSource.java | 16 ----------
 .../kafka/source/KafkaSourceTestUtils.java         |  6 ++++
 .../kafka/table/KafkaDynamicTableFactoryTest.java  | 34 +++++++++++++++++++++-
 3 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
index 8e12124..ab0fa13 100644
--- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
+++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
@@ -49,12 +49,9 @@ import org.apache.flink.table.types.DataType;
 import org.apache.flink.table.types.utils.DataTypeUtils;
 import org.apache.flink.util.Preconditions;
 
-import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.header.Header;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
 
@@ -77,8 +74,6 @@ import java.util.stream.Stream;
 public class KafkaDynamicSource
         implements ScanTableSource, SupportsReadingMetadata, SupportsWatermarkPushDown {
 
-    private static final Logger LOG = LoggerFactory.getLogger(KafkaDynamicSource.class);
-
     // --------------------------------------------------------------------------------------------
     // Mutable attributes
     // --------------------------------------------------------------------------------------------
@@ -389,17 +384,6 @@ public class KafkaDynamicSource
             kafkaSourceBuilder.setTopicPattern(topicPattern);
         }
 
-        // For compatibility with legacy source that is not validating group id
-        if (!properties.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) {
-            String generatedGroupId = "KafkaSource-" + tableIdentifier;
-            LOG.warn(
-                    "Property \"{}\" is required for offset commit but not set in table options. "
-                            + "Assigning \"{}\" as consumer group id",
-                    ConsumerConfig.GROUP_ID_CONFIG,
-                    generatedGroupId);
-            kafkaSourceBuilder.setGroupId(generatedGroupId);
-        }
-
         switch (startupMode) {
             case EARLIEST:
                 kafkaSourceBuilder.setStartingOffsets(OffsetsInitializer.earliest());
diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
index fce9591..572b77d 100644
--- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
+++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/connector/kafka/source/KafkaSourceTestUtils.java
@@ -19,6 +19,7 @@
 package org.apache.flink.connector.kafka.source;
 
 import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connector.kafka.source.reader.KafkaSourceReader;
 
 import java.util.Collection;
@@ -44,4 +45,9 @@ public class KafkaSourceTestUtils {
         return ((KafkaSourceReader<T>)
                 kafkaSource.createReader(sourceReaderContext, splitFinishedHook));
     }
+
+    /** Get configuration of KafkaSource. */
+    public static Configuration getKafkaSourceConfiguration(KafkaSource<?> kafkaSource) {
+        return kafkaSource.getConfiguration();
+    }
 }
diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
index 01af4b0..a0cc3cf 100644
--- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
+++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicTableFactoryTest.java
@@ -22,9 +22,13 @@ import org.apache.flink.api.common.serialization.DeserializationSchema;
 import org.apache.flink.api.common.serialization.SerializationSchema;
 import org.apache.flink.api.connector.sink.Sink;
 import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connector.base.DeliveryGuarantee;
 import org.apache.flink.connector.kafka.sink.KafkaSink;
 import org.apache.flink.connector.kafka.source.KafkaSource;
+import org.apache.flink.connector.kafka.source.KafkaSourceOptions;
+import org.apache.flink.connector.kafka.source.KafkaSourceTestUtils;
 import org.apache.flink.connector.kafka.source.enumerator.KafkaSourceEnumState;
 import org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit;
 import org.apache.flink.formats.avro.AvroRowDataSerializationSchema;
@@ -68,6 +72,7 @@ import org.apache.flink.util.TestLogger;
 
 import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
 
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -93,6 +98,7 @@ import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSou
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -350,6 +356,31 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
     }
 
     @Test
+    public void testTableSourceCommitOnCheckpointDisabled() {
+        final Map<String, String> modifiedOptions =
+                getModifiedOptions(
+                        getBasicSourceOptions(), options -> options.remove("properties.group.id"));
+        final DynamicTableSource tableSource = createTableSource(SCHEMA, modifiedOptions);
+
+        assertThat(tableSource, instanceOf(KafkaDynamicSource.class));
+        ScanTableSource.ScanRuntimeProvider providerWithoutGroupId =
+                ((KafkaDynamicSource) tableSource)
+                        .getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
+        assertThat(providerWithoutGroupId, instanceOf(DataStreamScanProvider.class));
+        final KafkaSource<?> kafkaSource = assertKafkaSource(providerWithoutGroupId);
+        final Configuration configuration =
+                KafkaSourceTestUtils.getKafkaSourceConfiguration(kafkaSource);
+
+        // Test offset commit on checkpoint should be disabled when do not set consumer group.
+        assertFalse(configuration.get(KafkaSourceOptions.COMMIT_OFFSETS_ON_CHECKPOINT));
+        assertFalse(
+                configuration.get(
+                        ConfigOptions.key(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)
+                                .booleanType()
+                                .noDefaultValue()));
+    }
+
+    @Test
     public void testTableSink() {
         final Map<String, String> modifiedOptions =
                 getModifiedOptions(
@@ -997,7 +1028,7 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
         return tableOptions;
     }
 
-    private void assertKafkaSource(ScanTableSource.ScanRuntimeProvider provider) {
+    private KafkaSource<?> assertKafkaSource(ScanTableSource.ScanRuntimeProvider provider) {
         assertThat(provider, instanceOf(DataStreamScanProvider.class));
         final DataStreamScanProvider dataStreamScanProvider = (DataStreamScanProvider) provider;
         final Transformation<RowData> transformation =
@@ -1010,5 +1041,6 @@ public class KafkaDynamicTableFactoryTest extends TestLogger {
                         (SourceTransformation<RowData, KafkaPartitionSplit, KafkaSourceEnumState>)
                                 transformation;
         assertThat(sourceTransformation.getSource(), instanceOf(KafkaSource.class));
+        return (KafkaSource<?>) sourceTransformation.getSource();
     }
 }