You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2025/08/30 04:25:18 UTC

(flink-connector-kafka) branch main updated: [FLINK-38289] Update to Flink 2.1

This is an automated email from the ASF dual-hosted git repository.

gaborgsomogyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-kafka.git


The following commit(s) were added to refs/heads/main by this push:
     new 032c9dd7 [FLINK-38289] Update to Flink 2.1
032c9dd7 is described below

commit 032c9dd7f94f3acb860c1e1d2e69a0513373bc2e
Author: Thomas Cooper <co...@tomcooper.dev>
AuthorDate: Sat Aug 30 05:25:13 2025 +0100

    [FLINK-38289] Update to Flink 2.1
---
 .github/workflows/push_pr.yml                      |   4 +-
 .../c0d94764-76a0-4c50-b617-70b1754c4612           |   6 +-
 .../connectors/kafka/table/KafkaDynamicSink.java   |   4 +-
 .../connectors/kafka/table/KafkaDynamicSource.java |   4 +-
 .../connectors/kafka/table/TableDataTypeUtils.java |  83 ++++++++++++++++
 .../kafka/table/TableDataTypeUtilsTest.java        | 104 +++++++++++++++++++++
 flink-python/tox.ini                               |   2 +-
 pom.xml                                            |   2 +-
 8 files changed, 197 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
index e4bd1e5e..4e5e9365 100644
--- a/.github/workflows/push_pr.yml
+++ b/.github/workflows/push_pr.yml
@@ -28,7 +28,7 @@ jobs:
   compile_and_test:
     strategy:
       matrix:
-        flink: [ 2.0.0 ]
+        flink: [ 2.1.0 ]
         jdk: [ '11, 17, 21' ]
     uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
     with:
@@ -37,7 +37,7 @@ jobs:
   python_test:
     strategy:
       matrix:
-        flink: [ 2.0.0 ]
+        flink: [ 2.1.0 ]
         jdk: [ '11, 17, 21' ]
     uses: apache/flink-connector-shared-utils/.github/workflows/python_ci.yml@ci_utils
     with:
diff --git a/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612 b/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
index a4256dac..236fade3 100644
--- a/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
+++ b/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
@@ -23,8 +23,6 @@ Method <org.apache.flink.connector.kafka.dynamic.source.reader.DynamicKafkaSourc
 Method <org.apache.flink.connector.kafka.dynamic.source.reader.DynamicKafkaSourceReader.syncAvailabilityHelperWithReaders()> calls method <org.apache.flink.streaming.runtime.io.MultipleFuturesAvailabilityHelper.anyOf(int, java.util.concurrent.CompletableFuture)> in (DynamicKafkaSourceReader.java:500)
 Method <org.apache.flink.connector.kafka.sink.ExactlyOnceKafkaWriter.getProducerPool()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ExactlyOnceKafkaWriter.java:0)
 Method <org.apache.flink.connector.kafka.sink.ExactlyOnceKafkaWriter.getTransactionalIdPrefix()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ExactlyOnceKafkaWriter.java:0)
-Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getBackchannel()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
-Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getCommittingProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
 Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getCoLocationGroupKey()> in (KafkaSink.java:178)
 Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getInputs()> in (KafkaSink.java:181)
 Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getOutputType()> in (KafkaSink.java:177)
@@ -34,6 +32,8 @@ Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(or
 Method <org.apache.flink.connector.kafka.sink.KafkaSink.getKafkaProducerConfig()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSink.java:0)
 Method <org.apache.flink.connector.kafka.sink.KafkaSinkBuilder.setRecordSerializer(org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema)> calls method <org.apache.flink.api.java.ClosureCleaner.clean(java.lang.Object, org.apache.flink.api.common.ExecutionConfig$ClosureCleanerLevel, boolean)> in (KafkaSinkBuilder.java:154)
 Method <org.apache.flink.connector.kafka.sink.KafkaWriter.getCurrentProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaWriter.java:0)
+Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getBackchannel()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
+Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getCommittingProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
 Method <org.apache.flink.connector.kafka.sink.internal.ProducerPoolImpl.getProducers()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ProducerPoolImpl.java:0)
 Method <org.apache.flink.connector.kafka.source.KafkaSource.createReader(org.apache.flink.api.connector.source.SourceReaderContext, java.util.function.Consumer)> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSource.java:0)
 Method <org.apache.flink.connector.kafka.source.KafkaSource.getConfiguration()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSource.java:0)
@@ -50,6 +50,4 @@ Method <org.apache.flink.connector.kafka.source.reader.KafkaSourceReader.getOffs
 Method <org.apache.flink.streaming.connectors.kafka.table.DynamicKafkaRecordSerializationSchema.createProjectedRow(org.apache.flink.table.data.RowData, org.apache.flink.types.RowKind, [Lorg.apache.flink.table.data.RowData$FieldGetter;)> has parameter of type <[Lorg.apache.flink.table.data.RowData$FieldGetter;> in (DynamicKafkaRecordSerializationSchema.java:0)
 Method <org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.createKeyFormatProjection(org.apache.flink.configuration.ReadableConfig, org.apache.flink.table.types.DataType)> calls method <org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldNames(org.apache.flink.table.types.logical.LogicalType)> in (KafkaConnectorOptionsUtil.java:520)
 Method <org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.createValueFormatProjection(org.apache.flink.configuration.ReadableConfig, org.apache.flink.table.types.DataType)> calls method <org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldCount(org.apache.flink.table.types.logical.LogicalType)> in (KafkaConnectorOptionsUtil.java:564)
-Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSink.createSerialization(org.apache.flink.table.connector.sink.DynamicTableSink$Context, org.apache.flink.table.connector.format.EncodingFormat, [I, java.lang.String)> calls method <org.apache.flink.table.types.utils.DataTypeUtils.stripRowPrefix(org.apache.flink.table.types.DataType, java.lang.String)> in (KafkaDynamicSink.java:408)
 Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSink.getFieldGetters(java.util.List, [I)> has return type <[Lorg.apache.flink.table.data.RowData$FieldGetter;> in (KafkaDynamicSink.java:0)
-Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSource.createDeserialization(org.apache.flink.table.connector.source.DynamicTableSource$Context, org.apache.flink.table.connector.format.DecodingFormat, [I, java.lang.String)> calls method <org.apache.flink.table.types.utils.DataTypeUtils.stripRowPrefix(org.apache.flink.table.types.DataType, java.lang.String)> in (KafkaDynamicSource.java:574)
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
index d6b1902e..4a1a4dd7 100644
--- a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
@@ -44,7 +44,6 @@ import org.apache.flink.table.data.MapData;
 import org.apache.flink.table.data.RowData;
 import org.apache.flink.table.types.DataType;
 import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.utils.DataTypeUtils;
 
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.common.header.Header;
@@ -405,7 +404,8 @@ public class KafkaDynamicSink implements DynamicTableSink, SupportsWritingMetada
         }
         DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
         if (prefix != null) {
-            physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+            physicalFormatDataType =
+                    TableDataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
         }
         return format.createRuntimeEncoder(context, physicalFormatDataType);
     }
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
index 79cd483c..8c86320e 100644
--- a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
@@ -49,7 +49,6 @@ import org.apache.flink.table.data.RowData;
 import org.apache.flink.table.data.StringData;
 import org.apache.flink.table.data.TimestampData;
 import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.utils.DataTypeUtils;
 import org.apache.flink.util.Preconditions;
 
 import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -571,7 +570,8 @@ public class KafkaDynamicSource
         }
         DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
         if (prefix != null) {
-            physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+            physicalFormatDataType =
+                    TableDataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
         }
         return format.createRuntimeDecoder(context, physicalFormatDataType);
     }
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java
new file mode 100644
index 00000000..6a598c6c
--- /dev/null
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java
@@ -0,0 +1,83 @@
+package org.apache.flink.streaming.connectors.kafka.table;
+
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.FieldsDataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.table.types.logical.LogicalTypeRoot.ROW;
+
+/**
+ * Utility class for manipulating {@link DataType} objects, particularly for table schemas. This
+ * class provides methods that were removed from the Flink API in version 2.1.0. See <a
+ * href="https://github.com/apache/flink/pull/26784">Flink PR 26784</a>.
+ */
+public class TableDataTypeUtils {
+
+    protected static final String STRIP_ROW_NO_ROW_ERROR_MSG = "Row data type expected.";
+    protected static final String RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG =
+            "Row length and new names must match.";
+
+    /**
+     * Removes a string prefix from the fields of the given row data type.
+     *
+     * @param dataType The row data type to modify.
+     * @param prefix The prefix to remove from the field names.
+     * @return A new DataType with the modified field names.
+     * @throws IllegalArgumentException if the provided dataType is not of ROW type.
+     */
+    public static DataType stripRowPrefix(DataType dataType, String prefix) {
+
+        if (!dataType.getLogicalType().is(ROW)) {
+            throw new IllegalArgumentException(STRIP_ROW_NO_ROW_ERROR_MSG);
+        }
+
+        final RowType rowType = (RowType) dataType.getLogicalType();
+        final List<String> newFieldNames =
+                rowType.getFieldNames().stream()
+                        .map(
+                                s -> {
+                                    if (s.startsWith(prefix)) {
+                                        return s.substring(prefix.length());
+                                    }
+                                    return s;
+                                })
+                        .collect(Collectors.toList());
+        final LogicalType newRowType = renameRowFields(rowType, newFieldNames);
+        return new FieldsDataType(
+                newRowType, dataType.getConversionClass(), dataType.getChildren());
+    }
+
+    /**
+     * Renames the fields of the given {@link RowType}.
+     *
+     * @param rowType The original RowType.
+     * @param newFieldNames The new field names to apply.
+     * @return A new RowType with the updated field names.
+     * @throws IllegalArgumentException if the number of new field names does not match the number
+     *     of fields in the original RowType.
+     */
+    public static RowType renameRowFields(RowType rowType, List<String> newFieldNames) {
+
+        if (!(rowType.getFieldCount() == newFieldNames.size())) {
+            throw new IllegalArgumentException(RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG);
+        }
+
+        final List<RowType.RowField> newFields =
+                IntStream.range(0, rowType.getFieldCount())
+                        .mapToObj(
+                                pos -> {
+                                    final RowType.RowField oldField = rowType.getFields().get(pos);
+                                    return new RowType.RowField(
+                                            newFieldNames.get(pos),
+                                            oldField.getType(),
+                                            oldField.getDescription().orElse(null));
+                                })
+                        .collect(Collectors.toList());
+        return new RowType(rowType.isNullable(), newFields);
+    }
+}
diff --git a/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java b/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java
new file mode 100644
index 00000000..8e95d6bc
--- /dev/null
+++ b/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java
@@ -0,0 +1,104 @@
+package org.apache.flink.streaming.connectors.kafka.table;
+
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.ROW;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link TableDataTypeUtils}. */
+public class TableDataTypeUtilsTest extends TestLogger {
+
+    @Test
+    public void testStripRowPrefix() {
+        DataType rowDataType =
+                ROW(
+                        FIELD("prefix_name", STRING()),
+                        FIELD("prefix_age", INT()),
+                        FIELD("address", STRING()));
+
+        DataType result = TableDataTypeUtils.stripRowPrefix(rowDataType, "prefix_");
+
+        RowType rowType = (RowType) result.getLogicalType();
+        List<String> fieldNames = rowType.getFieldNames();
+
+        assertThat(fieldNames).containsExactly("name", "age", "address");
+    }
+
+    @Test
+    public void testStripRowPrefixWithNoMatch() {
+        // Create a test row data type with no matching prefixes
+        DataType rowDataType =
+                ROW(FIELD("name", STRING()), FIELD("age", INT()), FIELD("address", STRING()));
+
+        DataType result = TableDataTypeUtils.stripRowPrefix(rowDataType, "nonexistent_");
+
+        // Field names should remain unchanged
+        RowType rowType = (RowType) result.getLogicalType();
+        List<String> fieldNames = rowType.getFieldNames();
+
+        assertThat(fieldNames).containsExactly("name", "age", "address");
+    }
+
+    @Test
+    public void testStripRowPrefixInvalidType() {
+        // Create a non-row data type
+        DataType nonRowType = STRING();
+
+        // Attempt to strip prefix should throw an exception
+        assertThatThrownBy(() -> TableDataTypeUtils.stripRowPrefix(nonRowType, "prefix_"))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining(TableDataTypeUtils.STRIP_ROW_NO_ROW_ERROR_MSG);
+    }
+
+    @Test
+    public void testRenameRowFields() {
+        List<RowType.RowField> fields =
+                Arrays.asList(
+                        new RowType.RowField("oldName1", new VarCharType(), null),
+                        new RowType.RowField("oldName2", new IntType(), "description"));
+        RowType rowType = new RowType(false, fields);
+
+        List<String> newFieldNames = Arrays.asList("newName1", "newName2");
+
+        RowType renamedType = TableDataTypeUtils.renameRowFields(rowType, newFieldNames);
+
+        List<String> resultFieldNames = renamedType.getFieldNames();
+        assertThat(resultFieldNames).containsExactly("newName1", "newName2");
+
+        assertThat(renamedType.getFields().get(0).getType()).isInstanceOf(VarCharType.class);
+        assertThat(renamedType.getFields().get(1).getType()).isInstanceOf(IntType.class);
+        assertThat(renamedType.getFields().get(1).getDescription().orElse(null))
+                .isEqualTo("description");
+    }
+
+    @Test
+    public void testRenameRowFieldsInvalidLength() {
+        List<RowType.RowField> fields =
+                Arrays.asList(
+                        new RowType.RowField("oldName1", new VarCharType(), null),
+                        new RowType.RowField("oldName2", new IntType(), null));
+        RowType rowType = new RowType(false, fields);
+
+        // Incorrect number of new field names
+        List<String> newFieldNames = Collections.singletonList("newName1");
+
+        // Rename with incorrect number of fields should throw an exception
+        assertThatThrownBy(() -> TableDataTypeUtils.renameRowFields(rowType, newFieldNames))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining(TableDataTypeUtils.RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG);
+    }
+}
diff --git a/flink-python/tox.ini b/flink-python/tox.ini
index c21c00f7..1f526fda 100644
--- a/flink-python/tox.ini
+++ b/flink-python/tox.ini
@@ -21,7 +21,7 @@
 # in multiple virtualenvs. This configuration file will run the
 # test suite on all supported python versions.
 # new environments will be excluded by default unless explicitly added to envlist.
-envlist = {py38, py39, py310}-cython
+envlist = {py39, py310, py311}-cython
 
 [testenv]
 whitelist_externals = /bin/bash
diff --git a/pom.xml b/pom.xml
index 7421fe2b..61b6c9f2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -53,7 +53,7 @@ under the License.
 
 		<!-- Main Dependencies -->
 		<confluent.version>7.9.2</confluent.version>
-		<flink.version>2.0.0</flink.version>
+		<flink.version>2.1.0</flink.version>
 		<kafka.version>4.0.0</kafka.version>
 
 		<!-- Other Dependencies -->