You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2025/08/30 04:25:18 UTC
(flink-connector-kafka) branch main updated: [FLINK-38289] Update to Flink 2.1
This is an automated email from the ASF dual-hosted git repository.
gaborgsomogyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-kafka.git
The following commit(s) were added to refs/heads/main by this push:
new 032c9dd7 [FLINK-38289] Update to Flink 2.1
032c9dd7 is described below
commit 032c9dd7f94f3acb860c1e1d2e69a0513373bc2e
Author: Thomas Cooper <co...@tomcooper.dev>
AuthorDate: Sat Aug 30 05:25:13 2025 +0100
[FLINK-38289] Update to Flink 2.1
---
.github/workflows/push_pr.yml | 4 +-
.../c0d94764-76a0-4c50-b617-70b1754c4612 | 6 +-
.../connectors/kafka/table/KafkaDynamicSink.java | 4 +-
.../connectors/kafka/table/KafkaDynamicSource.java | 4 +-
.../connectors/kafka/table/TableDataTypeUtils.java | 83 ++++++++++++++++
.../kafka/table/TableDataTypeUtilsTest.java | 104 +++++++++++++++++++++
flink-python/tox.ini | 2 +-
pom.xml | 2 +-
8 files changed, 197 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
index e4bd1e5e..4e5e9365 100644
--- a/.github/workflows/push_pr.yml
+++ b/.github/workflows/push_pr.yml
@@ -28,7 +28,7 @@ jobs:
compile_and_test:
strategy:
matrix:
- flink: [ 2.0.0 ]
+ flink: [ 2.1.0 ]
jdk: [ '11, 17, 21' ]
uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
with:
@@ -37,7 +37,7 @@ jobs:
python_test:
strategy:
matrix:
- flink: [ 2.0.0 ]
+ flink: [ 2.1.0 ]
jdk: [ '11, 17, 21' ]
uses: apache/flink-connector-shared-utils/.github/workflows/python_ci.yml@ci_utils
with:
diff --git a/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612 b/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
index a4256dac..236fade3 100644
--- a/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
+++ b/flink-connector-kafka/archunit-violations/c0d94764-76a0-4c50-b617-70b1754c4612
@@ -23,8 +23,6 @@ Method <org.apache.flink.connector.kafka.dynamic.source.reader.DynamicKafkaSourc
Method <org.apache.flink.connector.kafka.dynamic.source.reader.DynamicKafkaSourceReader.syncAvailabilityHelperWithReaders()> calls method <org.apache.flink.streaming.runtime.io.MultipleFuturesAvailabilityHelper.anyOf(int, java.util.concurrent.CompletableFuture)> in (DynamicKafkaSourceReader.java:500)
Method <org.apache.flink.connector.kafka.sink.ExactlyOnceKafkaWriter.getProducerPool()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ExactlyOnceKafkaWriter.java:0)
Method <org.apache.flink.connector.kafka.sink.ExactlyOnceKafkaWriter.getTransactionalIdPrefix()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ExactlyOnceKafkaWriter.java:0)
-Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getBackchannel()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
-Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getCommittingProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getCoLocationGroupKey()> in (KafkaSink.java:178)
Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getInputs()> in (KafkaSink.java:181)
Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(org.apache.flink.streaming.api.datastream.DataStream)> calls method <org.apache.flink.api.dag.Transformation.getOutputType()> in (KafkaSink.java:177)
@@ -34,6 +32,8 @@ Method <org.apache.flink.connector.kafka.sink.KafkaSink.addPostCommitTopology(or
Method <org.apache.flink.connector.kafka.sink.KafkaSink.getKafkaProducerConfig()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSink.java:0)
Method <org.apache.flink.connector.kafka.sink.KafkaSinkBuilder.setRecordSerializer(org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema)> calls method <org.apache.flink.api.java.ClosureCleaner.clean(java.lang.Object, org.apache.flink.api.common.ExecutionConfig$ClosureCleanerLevel, boolean)> in (KafkaSinkBuilder.java:154)
Method <org.apache.flink.connector.kafka.sink.KafkaWriter.getCurrentProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaWriter.java:0)
+Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getBackchannel()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
+Method <org.apache.flink.connector.kafka.sink.internal.KafkaCommitter.getCommittingProducer()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaCommitter.java:0)
Method <org.apache.flink.connector.kafka.sink.internal.ProducerPoolImpl.getProducers()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (ProducerPoolImpl.java:0)
Method <org.apache.flink.connector.kafka.source.KafkaSource.createReader(org.apache.flink.api.connector.source.SourceReaderContext, java.util.function.Consumer)> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSource.java:0)
Method <org.apache.flink.connector.kafka.source.KafkaSource.getConfiguration()> is annotated with <org.apache.flink.annotation.VisibleForTesting> in (KafkaSource.java:0)
@@ -50,6 +50,4 @@ Method <org.apache.flink.connector.kafka.source.reader.KafkaSourceReader.getOffs
Method <org.apache.flink.streaming.connectors.kafka.table.DynamicKafkaRecordSerializationSchema.createProjectedRow(org.apache.flink.table.data.RowData, org.apache.flink.types.RowKind, [Lorg.apache.flink.table.data.RowData$FieldGetter;)> has parameter of type <[Lorg.apache.flink.table.data.RowData$FieldGetter;> in (DynamicKafkaRecordSerializationSchema.java:0)
Method <org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.createKeyFormatProjection(org.apache.flink.configuration.ReadableConfig, org.apache.flink.table.types.DataType)> calls method <org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldNames(org.apache.flink.table.types.logical.LogicalType)> in (KafkaConnectorOptionsUtil.java:520)
Method <org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.createValueFormatProjection(org.apache.flink.configuration.ReadableConfig, org.apache.flink.table.types.DataType)> calls method <org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldCount(org.apache.flink.table.types.logical.LogicalType)> in (KafkaConnectorOptionsUtil.java:564)
-Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSink.createSerialization(org.apache.flink.table.connector.sink.DynamicTableSink$Context, org.apache.flink.table.connector.format.EncodingFormat, [I, java.lang.String)> calls method <org.apache.flink.table.types.utils.DataTypeUtils.stripRowPrefix(org.apache.flink.table.types.DataType, java.lang.String)> in (KafkaDynamicSink.java:408)
Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSink.getFieldGetters(java.util.List, [I)> has return type <[Lorg.apache.flink.table.data.RowData$FieldGetter;> in (KafkaDynamicSink.java:0)
-Method <org.apache.flink.streaming.connectors.kafka.table.KafkaDynamicSource.createDeserialization(org.apache.flink.table.connector.source.DynamicTableSource$Context, org.apache.flink.table.connector.format.DecodingFormat, [I, java.lang.String)> calls method <org.apache.flink.table.types.utils.DataTypeUtils.stripRowPrefix(org.apache.flink.table.types.DataType, java.lang.String)> in (KafkaDynamicSource.java:574)
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
index d6b1902e..4a1a4dd7 100644
--- a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSink.java
@@ -44,7 +44,6 @@ import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.utils.DataTypeUtils;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.header.Header;
@@ -405,7 +404,8 @@ public class KafkaDynamicSink implements DynamicTableSink, SupportsWritingMetada
}
DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
if (prefix != null) {
- physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+ physicalFormatDataType =
+ TableDataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
}
return format.createRuntimeEncoder(context, physicalFormatDataType);
}
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
index 79cd483c..8c86320e 100644
--- a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaDynamicSource.java
@@ -49,7 +49,6 @@ import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.data.TimestampData;
import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.utils.DataTypeUtils;
import org.apache.flink.util.Preconditions;
import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -571,7 +570,8 @@ public class KafkaDynamicSource
}
DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
if (prefix != null) {
- physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+ physicalFormatDataType =
+ TableDataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
}
return format.createRuntimeDecoder(context, physicalFormatDataType);
}
diff --git a/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java
new file mode 100644
index 00000000..6a598c6c
--- /dev/null
+++ b/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtils.java
@@ -0,0 +1,83 @@
+package org.apache.flink.streaming.connectors.kafka.table;
+
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.FieldsDataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.table.types.logical.LogicalTypeRoot.ROW;
+
+/**
+ * Utility class for manipulating {@link DataType} objects, particularly for table schemas. This
+ * class provides methods that were removed from the Flink API in version 2.1.0. See <a
+ * href="https://github.com/apache/flink/pull/26784">Flink PR 26784</a>.
+ */
+public class TableDataTypeUtils {
+
+ protected static final String STRIP_ROW_NO_ROW_ERROR_MSG = "Row data type expected.";
+ protected static final String RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG =
+ "Row length and new names must match.";
+
+ /**
+ * Removes a string prefix from the fields of the given row data type.
+ *
+ * @param dataType The row data type to modify.
+ * @param prefix The prefix to remove from the field names.
+ * @return A new DataType with the modified field names.
+ * @throws IllegalArgumentException if the provided dataType is not of ROW type.
+ */
+ public static DataType stripRowPrefix(DataType dataType, String prefix) {
+
+ if (!dataType.getLogicalType().is(ROW)) {
+ throw new IllegalArgumentException(STRIP_ROW_NO_ROW_ERROR_MSG);
+ }
+
+ final RowType rowType = (RowType) dataType.getLogicalType();
+ final List<String> newFieldNames =
+ rowType.getFieldNames().stream()
+ .map(
+ s -> {
+ if (s.startsWith(prefix)) {
+ return s.substring(prefix.length());
+ }
+ return s;
+ })
+ .collect(Collectors.toList());
+ final LogicalType newRowType = renameRowFields(rowType, newFieldNames);
+ return new FieldsDataType(
+ newRowType, dataType.getConversionClass(), dataType.getChildren());
+ }
+
+ /**
+ * Renames the fields of the given {@link RowType}.
+ *
+ * @param rowType The original RowType.
+ * @param newFieldNames The new field names to apply.
+ * @return A new RowType with the updated field names.
+ * @throws IllegalArgumentException if the number of new field names does not match the number
+ * of fields in the original RowType.
+ */
+ public static RowType renameRowFields(RowType rowType, List<String> newFieldNames) {
+
+ if (!(rowType.getFieldCount() == newFieldNames.size())) {
+ throw new IllegalArgumentException(RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG);
+ }
+
+ final List<RowType.RowField> newFields =
+ IntStream.range(0, rowType.getFieldCount())
+ .mapToObj(
+ pos -> {
+ final RowType.RowField oldField = rowType.getFields().get(pos);
+ return new RowType.RowField(
+ newFieldNames.get(pos),
+ oldField.getType(),
+ oldField.getDescription().orElse(null));
+ })
+ .collect(Collectors.toList());
+ return new RowType(rowType.isNullable(), newFields);
+ }
+}
diff --git a/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java b/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java
new file mode 100644
index 00000000..8e95d6bc
--- /dev/null
+++ b/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/table/TableDataTypeUtilsTest.java
@@ -0,0 +1,104 @@
+package org.apache.flink.streaming.connectors.kafka.table;
+
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.ROW;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link TableDataTypeUtils}. */
+public class TableDataTypeUtilsTest extends TestLogger {
+
+ @Test
+ public void testStripRowPrefix() {
+ DataType rowDataType =
+ ROW(
+ FIELD("prefix_name", STRING()),
+ FIELD("prefix_age", INT()),
+ FIELD("address", STRING()));
+
+ DataType result = TableDataTypeUtils.stripRowPrefix(rowDataType, "prefix_");
+
+ RowType rowType = (RowType) result.getLogicalType();
+ List<String> fieldNames = rowType.getFieldNames();
+
+ assertThat(fieldNames).containsExactly("name", "age", "address");
+ }
+
+ @Test
+ public void testStripRowPrefixWithNoMatch() {
+ // Create a test row data type with no matching prefixes
+ DataType rowDataType =
+ ROW(FIELD("name", STRING()), FIELD("age", INT()), FIELD("address", STRING()));
+
+ DataType result = TableDataTypeUtils.stripRowPrefix(rowDataType, "nonexistent_");
+
+ // Field names should remain unchanged
+ RowType rowType = (RowType) result.getLogicalType();
+ List<String> fieldNames = rowType.getFieldNames();
+
+ assertThat(fieldNames).containsExactly("name", "age", "address");
+ }
+
+ @Test
+ public void testStripRowPrefixInvalidType() {
+ // Create a non-row data type
+ DataType nonRowType = STRING();
+
+ // Attempt to strip prefix should throw an exception
+ assertThatThrownBy(() -> TableDataTypeUtils.stripRowPrefix(nonRowType, "prefix_"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(TableDataTypeUtils.STRIP_ROW_NO_ROW_ERROR_MSG);
+ }
+
+ @Test
+ public void testRenameRowFields() {
+ List<RowType.RowField> fields =
+ Arrays.asList(
+ new RowType.RowField("oldName1", new VarCharType(), null),
+ new RowType.RowField("oldName2", new IntType(), "description"));
+ RowType rowType = new RowType(false, fields);
+
+ List<String> newFieldNames = Arrays.asList("newName1", "newName2");
+
+ RowType renamedType = TableDataTypeUtils.renameRowFields(rowType, newFieldNames);
+
+ List<String> resultFieldNames = renamedType.getFieldNames();
+ assertThat(resultFieldNames).containsExactly("newName1", "newName2");
+
+ assertThat(renamedType.getFields().get(0).getType()).isInstanceOf(VarCharType.class);
+ assertThat(renamedType.getFields().get(1).getType()).isInstanceOf(IntType.class);
+ assertThat(renamedType.getFields().get(1).getDescription().orElse(null))
+ .isEqualTo("description");
+ }
+
+ @Test
+ public void testRenameRowFieldsInvalidLength() {
+ List<RowType.RowField> fields =
+ Arrays.asList(
+ new RowType.RowField("oldName1", new VarCharType(), null),
+ new RowType.RowField("oldName2", new IntType(), null));
+ RowType rowType = new RowType(false, fields);
+
+ // Incorrect number of new field names
+ List<String> newFieldNames = Collections.singletonList("newName1");
+
+ // Rename with incorrect number of fields should throw an exception
+ assertThatThrownBy(() -> TableDataTypeUtils.renameRowFields(rowType, newFieldNames))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(TableDataTypeUtils.RENAME_ROW_LENGTH_MISMATCH_ERROR_MSG);
+ }
+}
diff --git a/flink-python/tox.ini b/flink-python/tox.ini
index c21c00f7..1f526fda 100644
--- a/flink-python/tox.ini
+++ b/flink-python/tox.ini
@@ -21,7 +21,7 @@
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions.
# new environments will be excluded by default unless explicitly added to envlist.
-envlist = {py38, py39, py310}-cython
+envlist = {py39, py310, py311}-cython
[testenv]
whitelist_externals = /bin/bash
diff --git a/pom.xml b/pom.xml
index 7421fe2b..61b6c9f2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -53,7 +53,7 @@ under the License.
<!-- Main Dependencies -->
<confluent.version>7.9.2</confluent.version>
- <flink.version>2.0.0</flink.version>
+ <flink.version>2.1.0</flink.version>
<kafka.version>4.0.0</kafka.version>
<!-- Other Dependencies -->