You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2026/08/25 10:15:37 UTC
(hudi) branch master updated: feat(flink): support Flink 2.2 (#19655)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ef07f0f948c9 feat(flink): support Flink 2.2 (#19655)
ef07f0f948c9 is described below
commit ef07f0f948c9260eeaf07d5652ab08cb5401f6e1
Author: Shuo Cheng <nj...@gmail.com>
AuthorDate: Tue Aug 25 18:15:30 2026 +0800
feat(flink): support Flink 2.2 (#19655)
* feat(flink): support Flink 2.2
---
.github/workflows/bot.yml | 32 +-
.github/workflows/maven_artifact_validation.yml | 10 +-
.github/workflows/release_candidate_validation.yml | 4 +
README.md | 17 +-
azure-pipelines-20230430.yml | 4 +-
hudi-flink-datasource/hudi-flink2.2.x/pom.xml | 174 ++++
.../hudi/adapter/AbstractRichFunctionAdapter.java | 44 +
.../hudi/adapter/CollectionInputFormatAdapter.java | 34 +
.../adapter/DataStreamScanProviderAdapter.java | 36 +
.../adapter/DataStreamSinkProviderAdapter.java | 37 +
.../org/apache/hudi/adapter/DataTypeAdapter.java | 92 ++
.../apache/hudi/adapter/HiveCatalogConstants.java | 47 +
.../adapter/InputFormatSourceFunctionAdapter.java | 32 +
.../hudi/adapter/KeyedProcessFunctionAdapter.java | 43 +
.../apache/hudi/adapter/MaskingOutputAdapter.java | 79 ++
.../hudi/adapter/ProcessFunctionAdapter.java | 48 ++
.../hudi/adapter/RichSinkFunctionAdapter.java | 27 +
.../hudi/adapter/RichSourceFunctionAdapter.java | 44 +
.../java/org/apache/hudi/adapter/SinkAdapter.java | 37 +
.../apache/hudi/adapter/SinkFunctionAdapter.java | 27 +
.../hudi/adapter/SinkFunctionProviderAdapter.java | 28 +
.../apache/hudi/adapter/SourceFunctionAdapter.java | 27 +
.../adapter/SupportsPreWriteTopologyAdapter.java | 27 +
.../hudi/adapter/TableFunctionProviderAdapter.java | 33 +
.../hudi/adapter/TypeInformationAdapter.java | 35 +
.../main/java/org/apache/hudi/adapter/Utils.java | 180 ++++
.../adapter/YieldingOperatorFactoryAdapter.java | 27 +
.../table/format/cow/ParquetSplitReaderUtil.java | 950 +++++++++++++++++++++
.../table/format/cow/utils/BooleanArrayList.java | 69 ++
.../hudi/table/format/cow/utils/IntArrayList.java | 93 ++
.../hudi/table/format/cow/utils/LongArrayList.java | 83 ++
.../table/format/cow/utils/NestedPositionUtil.java | 209 +++++
.../table/format/cow/vector/HeapArrayVector.java | 101 +++
.../table/format/cow/vector/HeapDecimalVector.java | 39 +
.../format/cow/vector/HeapMapColumnVector.java | 121 +++
.../format/cow/vector/HeapRowColumnVector.java | 69 ++
.../format/cow/vector/ParquetDecimalVector.java | 227 +++++
.../cow/vector/position/CollectionPosition.java | 57 ++
.../cow/vector/position/LevelDelegation.java | 43 +
.../format/cow/vector/position/RowPosition.java | 45 +
.../cow/vector/reader/AbstractColumnReader.java | 322 +++++++
.../vector/reader/BaseVectorizedColumnReader.java | 305 +++++++
.../cow/vector/reader/EmptyColumnReader.java | 42 +
.../vector/reader/FixedLenBytesColumnReader.java | 84 ++
.../vector/reader/Int64TimestampColumnReader.java | 119 +++
.../cow/vector/reader/NestedColumnReader.java | 315 +++++++
.../vector/reader/NestedPrimitiveColumnReader.java | 636 ++++++++++++++
.../reader/ParquetColumnarRowSplitReader.java | 426 +++++++++
.../cow/vector/reader/ParquetDataColumnReader.java | 198 +++++
.../reader/ParquetDataColumnReaderFactory.java | 400 +++++++++
.../format/cow/vector/reader/RunLengthDecoder.java | 303 +++++++
.../table/format/cow/vector/type/ParquetField.java | 72 ++
.../format/cow/vector/type/ParquetGroupField.java | 59 ++
.../cow/vector/type/ParquetPrimitiveField.java | 55 ++
.../java/org/apache/hudi/utils/CatalogUtils.java | 36 +
.../java/org/apache/hudi/utils/DataTypeUtils.java | 33 +
.../org/apache/hudi/utils/RuntimeContextUtils.java | 62 ++
.../org/apache/hudi/utils/SourceContextUtils.java | 47 +
.../org/apache/hudi/utils/StateTtlConfigUtils.java | 32 +
.../java/org/apache/hudi/utils/StreamerUtils.java | 59 ++
.../java/typeutils/runtime/NoFetchingInput.java | 165 ++++
.../typeutils/runtime/kryo/KryoSerializer.java | 144 ++++
.../runtime/kryo/KryoSerializerSnapshot.java | 73 ++
.../apache/hudi/adapter/CollectOutputAdapter.java | 87 ++
.../hudi/adapter/DataTypeAdapterTestUtils.java | 32 +
.../apache/hudi/adapter/ExecutionAttemptUtil.java | 31 +
.../hudi/adapter/KeyedStateStoreAdapter.java | 53 ++
.../hudi/adapter/OperatorStateStoreAdapter.java | 38 +
.../org/apache/hudi/adapter/TestStreamConfigs.java | 35 +
.../org/apache/hudi/sink/utils/MockTaskInfo.java | 78 ++
.../cow/vector/TestHeapColumnVectorAccessors.java | 138 +++
.../cow/vector/TestParquetDecimalVector.java | 187 ++++
.../reader/TestParquetDataColumnReaderFactory.java | 270 ++++++
.../cow/vector/type/TestParquetGroupField.java | 132 +++
hudi-lakehouse/scripts/build-jars.sh | 2 +-
.../base/build_flink221hive313spark351.sh | 27 +
packaging/bundle-validation/ci_run.sh | 5 +
packaging/bundle-validation/validate.sh | 4 +-
pom.xml | 34 +-
scripts/release/deploy_staging_jars.sh | 1 +
scripts/release/validate_staged_bundles.sh | 2 +-
81 files changed, 8437 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 08db07cd2c12..1025838cf8b6 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -123,7 +123,7 @@ jobs:
include:
- scalaProfile: "scala-2.12"
sparkProfile: "spark3.5"
- flinkProfile: "flink2.1"
+ flinkProfile: "flink2.2"
steps:
- if: needs.changes.outputs.relevant == 'true'
@@ -189,7 +189,7 @@ jobs:
include:
- scalaProfile: "scala-2.12"
sparkProfile: "spark3.5"
- flinkProfile: "flink2.1"
+ flinkProfile: "flink2.2"
steps:
- if: needs.changes.outputs.relevant == 'true'
@@ -247,7 +247,7 @@ jobs:
include:
- scalaProfile: "scala-2.12"
sparkProfile: "spark3.5"
- flinkProfile: "flink2.1"
+ flinkProfile: "flink2.2"
env:
UT_MODULES: >-
@@ -675,7 +675,7 @@ jobs:
include:
- scalaProfile: "scala-2.12"
sparkProfile: "spark3.5"
- flinkProfile: "flink2.1"
+ flinkProfile: "flink2.2"
steps:
- if: needs.changes.outputs.relevant == 'true'
@@ -1072,7 +1072,7 @@ jobs:
# - flinkProfile: "flink2.0"
# flinkAvroVersion: "1.11.4"
# flinkParquetVersion: '1.14.4'
- - flinkProfile: "flink2.1"
+ - flinkProfile: "flink2.2"
flinkAvroVersion: "1.11.4"
flinkParquetVersion: '1.15.2'
steps:
@@ -1110,15 +1110,15 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
- if: ${{ endsWith(env.FLINK_PROFILE, '2.1') && needs.changes.outputs.relevant == 'true' }}
+ if: ${{ endsWith(env.FLINK_PROFILE, '2.2') && needs.changes.outputs.relevant == 'true' }}
run: |
mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
- name: Generate merged coverage report
- if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true'
+ if: always() && endsWith(matrix.flinkProfile, '2.2') && needs.changes.outputs.relevant == 'true'
run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
- name: Upload coverage to Codecov
- if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true'
+ if: always() && endsWith(matrix.flinkProfile, '2.2') && needs.changes.outputs.relevant == 'true'
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
with:
files: ./jacoco-report.xml
@@ -1132,7 +1132,7 @@ jobs:
strategy:
matrix:
include:
- - flinkProfile: "flink2.1"
+ - flinkProfile: "flink2.2"
flinkAvroVersion: "1.11.4"
flinkParquetVersion: '1.15.2'
steps:
@@ -1161,15 +1161,15 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
- if: ${{ endsWith(env.FLINK_PROFILE, '2.1') && needs.changes.outputs.relevant == 'true' }}
+ if: ${{ endsWith(env.FLINK_PROFILE, '2.2') && needs.changes.outputs.relevant == 'true' }}
run: |
mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
- name: Generate merged coverage report
- if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true'
+ if: always() && endsWith(matrix.flinkProfile, '2.2') && needs.changes.outputs.relevant == 'true'
run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
- name: Upload coverage to Codecov
- if: always() && endsWith(matrix.flinkProfile, '2.1') && needs.changes.outputs.relevant == 'true'
+ if: always() && endsWith(matrix.flinkProfile, '2.2') && needs.changes.outputs.relevant == 'true'
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
with:
files: ./jacoco-report.xml
@@ -1235,10 +1235,10 @@ jobs:
strategy:
matrix:
include:
- - scalaProfile: 'scala-2.13'
- flinkProfile: 'flink1.20'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink2.2'
flinkAvroVersion: '1.11.4'
- flinkParquetVersion: '1.13.1'
+ flinkParquetVersion: '1.15.2'
sparkProfile: 'spark3.5'
sparkRuntime: 'spark3.5.1'
# [CI-TRIM] to revisit for CI improvement
@@ -1436,7 +1436,7 @@ jobs:
matrix:
include:
- sparkProfile: 'spark3.5'
- flinkProfile: 'flink2.1'
+ flinkProfile: 'flink2.2'
sparkArchive: 'spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz'
steps:
- if: needs.changes.outputs.relevant == 'true'
diff --git a/.github/workflows/maven_artifact_validation.yml b/.github/workflows/maven_artifact_validation.yml
index 79ccc82413f7..bf8b84be8e12 100644
--- a/.github/workflows/maven_artifact_validation.yml
+++ b/.github/workflows/maven_artifact_validation.yml
@@ -23,7 +23,15 @@ jobs:
strategy:
matrix:
include:
- - scalaProfile: 'scala-2.13'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink2.2'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.1'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink2.1'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.1'
+ - scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.0'
sparkProfile: 'spark3.5'
sparkRuntime: 'spark3.5.1'
diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml
index cb8a976f6b47..5f769603b2e9 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -121,6 +121,10 @@ jobs:
flinkProfile: 'flink2.1'
sparkProfile: 'spark3.5'
sparkRuntime: 'spark3.5.1'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink2.2'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.1'
steps:
- uses: actions/checkout@v5
- name: Set up JDK 11
diff --git a/README.md b/README.md
index f2f97b40b2c4..f2ab0ca3caa3 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,7 @@ Prerequisites for building Apache Hudi:
```
# Checkout code and build
git clone https://github.com/apache/hudi.git && cd hudi
-mvn clean package -DskipTests -Dspark3.5 -Dflink2.1
+mvn clean package -DskipTests -Dspark3.5 -Dflink2.2
# Start command
spark-3.5.0-bin-hadoop3/bin/spark-shell \
@@ -151,10 +151,10 @@ mvn clean package -DskipTests -Dspark3.5 -Dscala-2.13 -pl packaging/hudi-spark-b
For example,
```
# Build against Spark 3.5.x
-mvn clean package -DskipTests -Dspark3.5 -Dflink2.1
+mvn clean package -DskipTests -Dspark3.5 -Dflink2.2
# Build against Spark 3.4.x
-mvn clean package -DskipTests -Dspark3.4 -Dflink2.1
+mvn clean package -DskipTests -Dspark3.4 -Dflink2.2
```
#### What about "spark-avro" module?
@@ -163,14 +163,15 @@ Starting from versions 0.11, Hudi no longer requires `spark-avro` to be specifie
### Build with different Flink versions
-The default Flink version supported is 2.1. The default Flink 2.1.x version, corresponding to the `flink2.1` profile, is 2.1.1.
+The default Flink version supported is 2.2. The default Flink 2.2.x version, corresponding to the `flink2.2` profile, is 2.2.1.
Flink is Scala-free since 1.15.x, there is no need to specify the Scala version for Flink 1.15.x and above versions.
Refer to the table below for building with different Flink and Scala versions.
| Maven build options | Expected Flink bundle jar name | Notes |
|:--------------------|:--------------------------------|:---------------------------------|
-| (empty) | hudi-flink2.1-bundle | For Flink 2.1 (default options) |
-| `-Dflink2.1` | hudi-flink2.1-bundle | For Flink 2.1 (same as default) |
+| (empty) | hudi-flink2.2-bundle | For Flink 2.2 (default options) |
+| `-Dflink2.2` | hudi-flink2.2-bundle | For Flink 2.2 (same as default) |
+| `-Dflink2.1` | hudi-flink2.1-bundle | For Flink 2.1 |
| `-Dflink2.0` | hudi-flink2.0-bundle | For Flink 2.0 |
| `-Dflink1.20` | hudi-flink1.20-bundle | For Flink 1.20 |
| `-Dflink1.19` | hudi-flink1.19-bundle | For Flink 1.19 |
@@ -178,8 +179,8 @@ Refer to the table below for building with different Flink and Scala versions.
For example,
```
-# Build against Flink 2.1.x
-mvn clean package -DskipTests -Dflink2.1
+# Build against Flink 2.2.x
+mvn clean package -DskipTests -Dflink2.2
```
## Running Tests
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 617bc4d9e418..cf9b84638edc 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -14,7 +14,7 @@
# limitations under the License.
# NOTE:
-# This config file defines how Azure CI runs tests with Spark 3.5 and Flink 2.1 profiles.
+# This config file defines how Azure CI runs tests with Spark 3.5 and Flink 2.2 profiles.
# PRs will need to keep in sync with master's version to trigger the CI runs.
# See scripts/jacoco/README.md for how aggregated code coverage report works
# across multiple modules.
@@ -101,7 +101,7 @@ parameters:
- '!packaging/hudi-utilities-slim-bundle'
variables:
- BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink2.1'
+ BUILD_PROFILES: '-Dscala-2.12 -Dspark3.5 -Dflink2.2'
PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
# The aether.* properties drive maven-resolver's native HTTP transport (Maven 3.9+); the
# maven.wagon.* ones are kept because this image's Maven version is not pinned and Maven 3.8
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/pom.xml b/hudi-flink-datasource/hudi-flink2.2.x/pom.xml
new file mode 100644
index 000000000000..1bfd665e0888
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/pom.xml
@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hudi-flink-datasource</artifactId>
+ <groupId>org.apache.hudi</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>hudi-flink2.2.x</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <main.basedir>${project.parent.parent.basedir}</main.basedir>
+ </properties>
+
+ <dependencies>
+ <!-- Lombok -->
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </dependency>
+
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- Hudi -->
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Flink -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java-bridge</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>${flink.connector.kafka.artifactId}</artifactId>
+ <version>${flink.connector.kafka.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-guava</artifactId>
+ <version>30.1.1-jre-14.0</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-runtime</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parquet</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${flink2.2.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-tests-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ <configuration>
+ <skip>false</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/AbstractRichFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/AbstractRichFunctionAdapter.java
new file mode 100644
index 000000000000..1f979fbadd55
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/AbstractRichFunctionAdapter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.functions.WithConfigurationOpenContext;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * Adapter clazz for {@code AbstractRichFunction} to support `open` with {@link Configuration}.
+ */
+public abstract class AbstractRichFunctionAdapter extends AbstractRichFunction {
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ Configuration configuration = new Configuration();
+ if (openContext instanceof WithConfigurationOpenContext) {
+ configuration = ((WithConfigurationOpenContext) openContext).getConfiguration();
+ }
+ open(configuration);
+ }
+
+ public void open(Configuration configuration) throws Exception {
+ // do nothing
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/CollectionInputFormatAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/CollectionInputFormatAdapter.java
new file mode 100644
index 000000000000..f59f8bd477d6
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/CollectionInputFormatAdapter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.streaming.api.legacy.io.CollectionInputFormat;
+
+import java.util.Collection;
+
+/**
+ * Adapter clazz for {@link CollectionInputFormat}.
+ */
+public class CollectionInputFormatAdapter<T> extends CollectionInputFormat<T> {
+
+ public CollectionInputFormatAdapter(Collection dataSet, TypeSerializer serializer) {
+ super(dataSet, serializer);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
new file mode 100644
index 000000000000..13cc0c2ac446
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.connector.ProviderContext;
+import org.apache.flink.table.connector.source.DataStreamScanProvider;
+import org.apache.flink.table.data.RowData;
+
+/**
+ * Adapter clazz for {@code DataStreamScanProvider}.
+ */
+public interface DataStreamScanProviderAdapter extends DataStreamScanProvider {
+ default DataStream<RowData> produceDataStream(ProviderContext providerContext, StreamExecutionEnvironment streamExecutionEnvironment) {
+ return produceDataStream(streamExecutionEnvironment);
+ }
+
+ DataStream<RowData> produceDataStream(StreamExecutionEnvironment streamExecutionEnvironment);
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
new file mode 100644
index 000000000000..349f60f30acf
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.table.connector.ProviderContext;
+import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
+import org.apache.flink.table.data.RowData;
+
+/**
+ * Adapter clazz for {@code DataStreamSinkProvider}.
+ */
+public interface DataStreamSinkProviderAdapter extends DataStreamSinkProvider {
+ DataStreamSink<?> consumeDataStream(DataStream<RowData> dataStream);
+
+ @Override
+ default DataStreamSink<?> consumeDataStream(ProviderContext providerContext, DataStream<RowData> dataStream) {
+ return consumeDataStream(dataStream);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
new file mode 100644
index 000000000000..47f87a62dde9
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.hudi.common.util.Option;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.types.variant.BinaryVariant;
+import org.apache.flink.types.variant.Variant;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+
+import java.lang.reflect.Method;
+
+/**
+ * Adapter utils to provide {@code DataType} utilities.
+ */
+public class DataTypeAdapter {
+
+ /**
+ * The Parquet Variant binary format specification version passed to
+ * {@code LogicalTypeAnnotation.variantType(byte)}. Version 1 is the initial spec
+ * defined by the Parquet Variant proposal (parquet-format 2.11.0 / parquet-java 1.16.0).
+ */
+ private static final byte VARIANT_SPEC_VERSION = 1;
+
+ /**
+ * Cached VARIANT annotation resolved via reflection. Empty if parquet-java
+ * on the classpath predates {@code LogicalTypeAnnotation.variantType()} (< 1.16.0).
+ */
+ private static final Option<LogicalTypeAnnotation> VARIANT_ANNOTATION = resolveVariantAnnotation();
+
+ private static Option<LogicalTypeAnnotation> resolveVariantAnnotation() {
+ try {
+ Method factory = LogicalTypeAnnotation.class.getMethod("variantType", byte.class);
+ return Option.of((LogicalTypeAnnotation) factory.invoke(null, VARIANT_SPEC_VERSION));
+ } catch (Exception e) {
+ return Option.empty();
+ }
+ }
+
+ /**
+ * Returns the Parquet VARIANT {@link LogicalTypeAnnotation} if parquet-java 1.16.0+ is on the
+ * classpath, or empty if the annotation class is unavailable.
+ */
+ public static Option<LogicalTypeAnnotation> variantParquetAnnotation() {
+ return VARIANT_ANNOTATION;
+ }
+
+ public static Variant getVariant(RowData rowData, int pos) {
+ return rowData.getVariant(pos);
+ }
+
+ public static Object createVariant(byte[] value, byte[] metadata) {
+ return new BinaryVariant(value, metadata);
+ }
+
+ public static boolean isVariantType(LogicalType logicalType) {
+ return logicalType.getTypeRoot() == LogicalTypeRoot.VARIANT;
+ }
+
+ public static DataType createVariantType() {
+ return DataTypes.VARIANT();
+ }
+
+ public static byte[] getVariantMetadata(Object obj) {
+ return ((BinaryVariant) obj).getMetadata();
+ }
+
+ public static byte[] getVariantValue(Object obj) {
+ return ((BinaryVariant) obj).getValue();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
new file mode 100644
index 000000000000..3e9c03419949
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+/**
+ * Constants for Hive Catalog.
+ */
+public class HiveCatalogConstants {
+
+ // -----------------------------------------------------------------------------------
+ // Constants for ALTER DATABASE
+ // -----------------------------------------------------------------------------------
+ public static final String ALTER_DATABASE_OP = "hive.alter.database.op";
+
+ public static final String DATABASE_LOCATION_URI = "hive.database.location-uri";
+
+ public static final String DATABASE_OWNER_NAME = "hive.database.owner.name";
+
+ public static final String DATABASE_OWNER_TYPE = "hive.database.owner.type";
+
+ public static final String ROLE_OWNER = "role";
+
+ public static final String USER_OWNER = "user";
+
+ /** Type of ALTER DATABASE operation. */
+ public enum AlterHiveDatabaseOp {
+ CHANGE_PROPS,
+ CHANGE_LOCATION,
+ CHANGE_OWNER
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/InputFormatSourceFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/InputFormatSourceFunctionAdapter.java
new file mode 100644
index 000000000000..01b045a23d23
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/InputFormatSourceFunctionAdapter.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.functions.source.legacy.InputFormatSourceFunction;
+
+/**
+ * Adapter clazz for {@link InputFormatSourceFunction}.
+ */
+public class InputFormatSourceFunctionAdapter<OUT> extends InputFormatSourceFunction<OUT> {
+ public InputFormatSourceFunctionAdapter(InputFormat format, TypeInformation typeInfo) {
+ super(format, typeInfo);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/KeyedProcessFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/KeyedProcessFunctionAdapter.java
new file mode 100644
index 000000000000..4d16bbfa936f
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/KeyedProcessFunctionAdapter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.functions.WithConfigurationOpenContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+
+/**
+ * Adapter clazz for {@link KeyedProcessFunction}.
+ */
+public abstract class KeyedProcessFunctionAdapter<K, I, O> extends KeyedProcessFunction<K, I, O> {
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ Configuration configuration = new Configuration();
+ if (openContext instanceof WithConfigurationOpenContext) {
+ configuration = ((WithConfigurationOpenContext) openContext).getConfiguration();
+ }
+ open(configuration);
+ }
+
+ public void open(Configuration configuration) throws Exception {
+ // do nothing
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
new file mode 100644
index 000000000000..56e1ce7991f7
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.runtime.event.WatermarkEvent;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+import org.apache.flink.util.OutputTag;
+
+/** Adapter class for {@code Output} to handle async compaction/clustering service thread safe issues */
+public class MaskingOutputAdapter<OUT> implements Output<StreamRecord<OUT>> {
+
+ private final Output<StreamRecord<OUT>> output;
+
+ public MaskingOutputAdapter(Output<StreamRecord<OUT>> output) {
+ this.output = output;
+ }
+
+ @Override
+ public void emitWatermark(Watermark watermark) {
+ // For thread safe, not to propagate the watermark
+ }
+
+ @Override
+ public void emitLatencyMarker(LatencyMarker latencyMarker) {
+ // For thread safe, not to propagate latency marker
+ }
+
+ @Override
+ public void emitRecordAttributes(RecordAttributes recordAttributes) {
+ // For thread safe, not to propagate watermark status
+ }
+
+ @Override
+ public void emitWatermark(WatermarkEvent watermarkEvent) {
+ // For thread safe, not to propagate the watermark
+ }
+
+ @Override
+ public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {
+ // For thread safe, not to propagate watermark status
+ }
+
+ @Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> streamRecord) {
+ this.output.collect(outputTag, streamRecord);
+ }
+
+ @Override
+ public void collect(StreamRecord<OUT> outStreamRecord) {
+ this.output.collect(outStreamRecord);
+ }
+
+ @Override
+ public void close() {
+ this.output.close();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/ProcessFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/ProcessFunctionAdapter.java
new file mode 100644
index 000000000000..1e5ec90e9f26
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/ProcessFunctionAdapter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.functions.WithConfigurationOpenContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+
+/**
+ * Adapter clazz for {@link ProcessFunction}.
+ */
+public abstract class ProcessFunctionAdapter<I, O> extends ProcessFunction<I, O> {
+
+ public ProcessFunctionAdapter() {
+ super();
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ Configuration configuration = new Configuration();
+ if (openContext instanceof WithConfigurationOpenContext) {
+ configuration = ((WithConfigurationOpenContext) openContext).getConfiguration();
+ }
+ open(configuration);
+ }
+
+ public void open(Configuration configuration) throws Exception {
+ // do nothing
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSinkFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSinkFunctionAdapter.java
new file mode 100644
index 000000000000..7a40383dde5b
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSinkFunctionAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.functions.sink.legacy.RichSinkFunction;
+
+/**
+ * Adapter clazz for {@link RichSinkFunction}.
+ */
+public abstract class RichSinkFunctionAdapter<I> extends RichSinkFunction<I> {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSourceFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSourceFunctionAdapter.java
new file mode 100644
index 000000000000..cc776fc68976
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/RichSourceFunctionAdapter.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.functions.WithConfigurationOpenContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.legacy.RichSourceFunction;
+
+/**
+ * Adapter clazz for {@link RichSourceFunction}.
+ */
+public abstract class RichSourceFunctionAdapter<OUT> extends RichSourceFunction<OUT> {
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ Configuration configuration = new Configuration();
+ if (openContext instanceof WithConfigurationOpenContext) {
+ configuration = ((WithConfigurationOpenContext) openContext).getConfiguration();
+ }
+ open(configuration);
+ }
+
+ public void open(Configuration configuration) throws Exception {
+ // do nothing
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkAdapter.java
new file mode 100644
index 000000000000..674d7fb37d6b
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkAdapter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+
+import java.io.IOException;
+
+/**
+ * Adapter clazz for {@link Sink}.
+ */
+public interface SinkAdapter<I> extends Sink<I> {
+
+ default SinkWriter<I> createWriter(WriterInitContext context) throws IOException {
+ return createWriter();
+ }
+
+ SinkWriter<I> createWriter() throws IOException;
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionAdapter.java
new file mode 100644
index 000000000000..211608b2a943
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction;
+
+/**
+ * Adapter clazz for {@link SinkFunction}.
+ */
+public interface SinkFunctionAdapter<I> extends SinkFunction<I> {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionProviderAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionProviderAdapter.java
new file mode 100644
index 000000000000..162812b68bb0
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SinkFunctionProviderAdapter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.table.connector.sink.legacy.SinkFunctionProvider;
+
+/**
+ * Adapter clazz for {@link SinkFunctionProvider}.
+ */
+public interface SinkFunctionProviderAdapter extends SinkFunctionProvider {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SourceFunctionAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SourceFunctionAdapter.java
new file mode 100644
index 000000000000..54d4c2f79138
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SourceFunctionAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.functions.source.legacy.SourceFunction;
+
+/**
+ * Adapter clazz for {@link SourceFunction}.
+ */
+public interface SourceFunctionAdapter<T> extends SourceFunction<T> {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SupportsPreWriteTopologyAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SupportsPreWriteTopologyAdapter.java
new file mode 100644
index 000000000000..c69012fa55b7
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/SupportsPreWriteTopologyAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.connector.sink2.SupportsPreWriteTopology;
+
+/**
+ * Adapter clazz for {@link SupportsPreWriteTopology}.
+ */
+public interface SupportsPreWriteTopologyAdapter<InputT> extends SupportsPreWriteTopology<InputT> {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TableFunctionProviderAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TableFunctionProviderAdapter.java
new file mode 100644
index 000000000000..297a2fa877df
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TableFunctionProviderAdapter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.table.functions.TableFunction;
+import org.apache.flink.table.legacy.connector.source.TableFunctionProvider;
+
+/**
+ * Adapter clazz for {@link TableFunctionProvider}.
+ */
+public interface TableFunctionProviderAdapter<T> extends TableFunctionProvider<T> {
+ static <T> TableFunctionProvider<T> of(TableFunction<T> tableFunction) {
+ return () -> {
+ return tableFunction;
+ };
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TypeInformationAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TypeInformationAdapter.java
new file mode 100644
index 000000000000..2aaa385236d4
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/TypeInformationAdapter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.serialization.SerializerConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+
+/**
+ * Adapter clazz for {@link TypeInformation}.
+ */
+public abstract class TypeInformationAdapter<T> extends TypeInformation<T> {
+ @Override
+ public TypeSerializer<T> createSerializer(SerializerConfig config) {
+ return createSerializer();
+ }
+
+ public abstract TypeSerializer<T> createSerializer();
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/Utils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/Utils.java
new file mode 100644
index 000000000000..1692d0c20bf5
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/Utils.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.hudi.common.schema.internal.InternalSchema;
+import org.apache.hudi.common.schema.internal.Type;
+import org.apache.hudi.common.schema.internal.action.InternalSchemaChangeApplier;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieNotSupportedException;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.core.memory.ManagedMemoryUseCase;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.table.api.config.ExecutionConfigOptions;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.TableChange;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.runtime.generated.NormalizedKeyComputer;
+import org.apache.flink.table.runtime.generated.RecordComparator;
+import org.apache.flink.table.runtime.operators.sort.BinaryExternalSorter;
+import org.apache.flink.table.runtime.typeutils.AbstractRowDataSerializer;
+import org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer;
+import org.apache.flink.table.types.logical.LogicalType;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType.AFTER;
+import static org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType.FIRST;
+import static org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType.NO_OPERATION;
+
+/**
+ * Adapter utils.
+ */
+public class Utils {
+ public static FactoryUtil.DefaultDynamicTableContext getTableContext(
+ ObjectIdentifier tablePath,
+ ResolvedCatalogTable catalogTable,
+ ReadableConfig conf) {
+ return new FactoryUtil.DefaultDynamicTableContext(tablePath, catalogTable,
+ Collections.emptyMap(), conf, Thread.currentThread().getContextClassLoader(), false);
+ }
+
+ public static BinaryExternalSorter getBinaryExternalSorter(
+ final Object owner,
+ MemoryManager memoryManager,
+ long reservedMemorySize,
+ IOManager ioManager,
+ AbstractRowDataSerializer<RowData> inputSerializer,
+ BinaryRowDataSerializer serializer,
+ NormalizedKeyComputer normalizedKeyComputer,
+ RecordComparator comparator,
+ Configuration conf) {
+ return new BinaryExternalSorter(owner, memoryManager, reservedMemorySize,
+ ioManager, inputSerializer, serializer, normalizedKeyComputer, comparator,
+ conf.get(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES),
+ conf.get(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED),
+ (int) conf.get(
+ ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE).getBytes(),
+ conf.get(ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED));
+ }
+
+ public static InternalSchema applyTableChange(InternalSchema oldSchema, List changes, Function<LogicalType, Type> convertFunc) {
+ InternalSchema newSchema = oldSchema;
+ for (Object change : changes) {
+ TableChange tableChange = (TableChange) change;
+ newSchema = applyTableChange(newSchema, tableChange, convertFunc);
+ }
+ return newSchema;
+ }
+
+ /**
+ * Computes the managed memory size available for the given stream operator.
+ * <p>
+ * This method retrieves the memory manager from the operator's execution environment
+ * and calculates the memory size allocated for the operator based on the configured
+ * managed memory fraction and the task manager's memory settings.
+ *
+ * @param operator the stream operator for which to compute managed memory
+ * @return the computed managed memory size in bytes
+ */
+ public static long computeManagedMemory(AbstractStreamOperator<?> operator) {
+ final Environment environment = operator.getContainingTask().getEnvironment();
+ return environment
+ .getMemoryManager()
+ .computeMemorySize(
+ operator.getOperatorConfig()
+ .getManagedMemoryFractionOperatorUseCaseOfSlot(
+ ManagedMemoryUseCase.OPERATOR,
+ environment.getJobConfiguration(),
+ environment.getTaskManagerInfo().getConfiguration(),
+ environment.getUserCodeClassLoader().asClassLoader()));
+ }
+
+ private static InternalSchema applyTableChange(InternalSchema oldSchema, TableChange change, Function<LogicalType, Type> convertFunc) {
+ InternalSchemaChangeApplier changeApplier = new InternalSchemaChangeApplier(oldSchema);
+ if (change instanceof TableChange.AddColumn) {
+ if (((TableChange.AddColumn) change).getColumn().isPhysical()) {
+ TableChange.AddColumn add = (TableChange.AddColumn) change;
+ Column column = add.getColumn();
+ String colName = column.getName();
+ Type colType = convertFunc.apply(column.getDataType().getLogicalType());
+ String comment = column.getComment().orElse(null);
+ Pair<org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType, String> colPositionPair =
+ parseColumnPosition(add.getPosition());
+ return changeApplier.applyAddChange(
+ colName, colType, comment, colPositionPair.getRight(), colPositionPair.getLeft());
+ } else {
+ throw new HoodieNotSupportedException("Add non-physical column is not supported yet.");
+ }
+ } else if (change instanceof TableChange.DropColumn) {
+ TableChange.DropColumn drop = (TableChange.DropColumn) change;
+ return changeApplier.applyDeleteChange(drop.getColumnName());
+ } else if (change instanceof TableChange.ModifyColumnName) {
+ TableChange.ModifyColumnName modify = (TableChange.ModifyColumnName) change;
+ String oldColName = modify.getOldColumnName();
+ String newColName = modify.getNewColumnName();
+ return changeApplier.applyRenameChange(oldColName, newColName);
+ } else if (change instanceof TableChange.ModifyPhysicalColumnType) {
+ TableChange.ModifyPhysicalColumnType modify = (TableChange.ModifyPhysicalColumnType) change;
+ String colName = modify.getOldColumn().getName();
+ Type newColType = convertFunc.apply(modify.getNewType().getLogicalType());
+ return changeApplier.applyColumnTypeChange(colName, newColType);
+ } else if (change instanceof TableChange.ModifyColumnPosition) {
+ TableChange.ModifyColumnPosition modify = (TableChange.ModifyColumnPosition) change;
+ String colName = modify.getOldColumn().getName();
+ Pair<org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType, String> colPositionPair =
+ parseColumnPosition(modify.getNewPosition());
+ return changeApplier.applyReOrderColPositionChange(
+ colName, colPositionPair.getRight(), colPositionPair.getLeft());
+ } else if (change instanceof TableChange.ModifyColumnComment) {
+ TableChange.ModifyColumnComment modify = (TableChange.ModifyColumnComment) change;
+ String colName = modify.getOldColumn().getName();
+ String comment = modify.getNewComment();
+ return changeApplier.applyColumnCommentChange(colName, comment);
+ } else if (change instanceof TableChange.ResetOption || change instanceof TableChange.SetOption) {
+ return oldSchema;
+ } else {
+ throw new HoodieNotSupportedException(change.getClass().getSimpleName() + " is not supported.");
+ }
+ }
+
+ private static Pair<org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType, String> parseColumnPosition(TableChange.ColumnPosition colPosition) {
+ org.apache.hudi.common.schema.internal.action.TableChange.ColumnPositionChange.ColumnPositionType positionType;
+ String position = "";
+ if (colPosition instanceof TableChange.First) {
+ positionType = FIRST;
+ } else if (colPosition instanceof TableChange.After) {
+ positionType = AFTER;
+ position = ((TableChange.After) colPosition).column();
+ } else {
+ positionType = NO_OPERATION;
+ }
+ return Pair.of(positionType, position);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/YieldingOperatorFactoryAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/YieldingOperatorFactoryAdapter.java
new file mode 100644
index 000000000000..94937af07e11
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/adapter/YieldingOperatorFactoryAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.streaming.api.operators.legacy.YieldingOperatorFactory;
+
+/**
+ * Adapter clazz for {@link YieldingOperatorFactory}.
+ */
+public interface YieldingOperatorFactoryAdapter<OUT> extends YieldingOperatorFactory<OUT> {
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
new file mode 100644
index 000000000000..756e7e7069da
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -0,0 +1,950 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow;
+
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.table.format.cow.vector.HeapArrayVector;
+import org.apache.hudi.table.format.cow.vector.HeapDecimalVector;
+import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector;
+import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector;
+import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader;
+import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader;
+import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader;
+import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader;
+import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField;
+
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.ByteColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.BytesColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.DoubleColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.FloatColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.IntColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.LongColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.ShortColumnReader;
+import org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LocalZonedTimestampType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.VarBinaryType;
+import org.apache.flink.table.types.logical.VariantType;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.ParquetRuntimeException;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReadStore;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.filter.UnboundRecordFilter;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.io.ColumnIO;
+import org.apache.parquet.io.GroupColumnIO;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.PrimitiveColumnIO;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.InvalidSchemaException;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.utils.DateTimeUtils.toInternal;
+import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
+import static org.apache.parquet.Preconditions.checkArgument;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+
+/**
+ * Util for generating {@link ParquetColumnarRowSplitReader}.
+ *
+ * <p>Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive
+ * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader},
+ * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For
+ * nested types (ARRAY / MAP / MULTISET / ROW / VARIANT) we build a {@link ParquetField} tree once
+ * per split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to
+ * {@link NestedColumnReader}.
+ *
+ * <p>Schema evolution: missing top-level fields are still handled by the caller
+ * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields
+ * <em>inside</em> a Row are handled here — {@link #constructField} returns {@code null} for a
+ * child that isn't physically present, and the corresponding child in the pre-allocated vector
+ * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can
+ * passthrough the slot (see {@link NestedColumnReader#readToVector}).
+ */
+public class ParquetSplitReaderUtil {
+
+ /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */
+ public static ParquetColumnarRowSplitReader genPartColumnarRowReader(
+ boolean utcTimestamp,
+ boolean caseSensitive,
+ Configuration conf,
+ String[] fullFieldNames,
+ DataType[] fullFieldTypes,
+ Map<String, Object> partitionSpec,
+ int[] selectedFields,
+ int batchSize,
+ Path path,
+ long splitStart,
+ long splitLength,
+ FilterPredicate filterPredicate,
+ UnboundRecordFilter recordFilter) throws IOException {
+
+ ValidationUtils.checkState(Arrays.stream(selectedFields).noneMatch(x -> x == -1),
+ "One or more specified columns does not exist in the hudi table.");
+
+ List<String> selNonPartNames = Arrays.stream(selectedFields)
+ .mapToObj(i -> fullFieldNames[i])
+ .filter(n -> !partitionSpec.containsKey(n))
+ .collect(Collectors.toList());
+
+ int[] selParquetFields = Arrays.stream(selectedFields)
+ .filter(i -> !partitionSpec.containsKey(fullFieldNames[i]))
+ .toArray();
+
+ ParquetColumnarRowSplitReader.ColumnBatchGenerator gen = readVectors -> {
+ // create and initialize the row batch
+ ColumnVector[] vectors = new ColumnVector[selectedFields.length];
+ for (int i = 0; i < vectors.length; i++) {
+ String name = fullFieldNames[selectedFields[i]];
+ LogicalType type = fullFieldTypes[selectedFields[i]].getLogicalType();
+ vectors[i] = createVector(readVectors, selNonPartNames, name, type, partitionSpec, batchSize);
+ }
+ return new VectorizedColumnBatch(vectors);
+ };
+
+ return new ParquetColumnarRowSplitReader(
+ utcTimestamp,
+ caseSensitive,
+ conf,
+ Arrays.stream(selParquetFields)
+ .mapToObj(i -> fullFieldTypes[i].getLogicalType())
+ .toArray(LogicalType[]::new),
+ selNonPartNames.toArray(new String[0]),
+ gen,
+ batchSize,
+ new org.apache.hadoop.fs.Path(path.toUri()),
+ splitStart,
+ splitLength,
+ filterPredicate,
+ recordFilter);
+ }
+
+ private static ColumnVector createVector(
+ ColumnVector[] readVectors,
+ List<String> selNonPartNames,
+ String name,
+ LogicalType type,
+ Map<String, Object> partitionSpec,
+ int batchSize) {
+ if (partitionSpec.containsKey(name)) {
+ return createVectorFromConstant(type, partitionSpec.get(name), batchSize);
+ }
+ ColumnVector readVector = readVectors[selNonPartNames.indexOf(name)];
+ if (readVector == null) {
+ // when the read vector is null, use a constant null vector instead
+ readVector = createVectorFromConstant(type, null, batchSize);
+ }
+ return readVector;
+ }
+
+ /**
+ * Builds a constant-filled column vector for either a partition column (non-null value) or a
+ * missing-column slot (null value). Used both at the batch-generator level for partition
+ * injection and at the row-reader level for fields absent from the Parquet file.
+ */
+ public static ColumnVector createVectorFromConstant(
+ LogicalType type, Object value, int batchSize) {
+ switch (type.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ HeapBytesVector bsv = new HeapBytesVector(batchSize);
+ if (value == null) {
+ bsv.fillWithNulls();
+ } else {
+ bsv.fill(value instanceof byte[]
+ ? (byte[]) value
+ : getUTF8Bytes(value.toString()));
+ }
+ return bsv;
+ case BOOLEAN:
+ HeapBooleanVector bv = new HeapBooleanVector(batchSize);
+ if (value == null) {
+ bv.fillWithNulls();
+ } else {
+ bv.fill((boolean) value);
+ }
+ return bv;
+ case TINYINT:
+ HeapByteVector byteVector = new HeapByteVector(batchSize);
+ if (value == null) {
+ byteVector.fillWithNulls();
+ } else {
+ byteVector.fill(((Number) value).byteValue());
+ }
+ return byteVector;
+ case SMALLINT:
+ HeapShortVector sv = new HeapShortVector(batchSize);
+ if (value == null) {
+ sv.fillWithNulls();
+ } else {
+ sv.fill(((Number) value).shortValue());
+ }
+ return sv;
+ case INTEGER:
+ HeapIntVector iv = new HeapIntVector(batchSize);
+ if (value == null) {
+ iv.fillWithNulls();
+ } else {
+ iv.fill(((Number) value).intValue());
+ }
+ return iv;
+ case BIGINT:
+ HeapLongVector lv = new HeapLongVector(batchSize);
+ if (value == null) {
+ lv.fillWithNulls();
+ } else {
+ lv.fill(((Number) value).longValue());
+ }
+ return lv;
+ case DECIMAL:
+ HeapDecimalVector decv = new HeapDecimalVector(batchSize);
+ if (value == null) {
+ decv.fillWithNulls();
+ } else {
+ DecimalType decimalType = (DecimalType) type;
+ int precision = decimalType.getPrecision();
+ int scale = decimalType.getScale();
+ DecimalData decimal = Preconditions.checkNotNull(
+ DecimalData.fromBigDecimal((BigDecimal) value, precision, scale));
+ decv.fill(decimal.toUnscaledBytes());
+ }
+ return decv;
+ case FLOAT:
+ HeapFloatVector fv = new HeapFloatVector(batchSize);
+ if (value == null) {
+ fv.fillWithNulls();
+ } else {
+ fv.fill(((Number) value).floatValue());
+ }
+ return fv;
+ case DOUBLE:
+ HeapDoubleVector dv = new HeapDoubleVector(batchSize);
+ if (value == null) {
+ dv.fillWithNulls();
+ } else {
+ dv.fill(((Number) value).doubleValue());
+ }
+ return dv;
+ case DATE:
+ if (value instanceof LocalDate) {
+ value = Date.valueOf((LocalDate) value);
+ }
+ return createVectorFromConstant(
+ new IntType(),
+ value == null ? null : toInternal((Date) value),
+ batchSize);
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HeapTimestampVector tv = new HeapTimestampVector(batchSize);
+ if (value == null) {
+ tv.fillWithNulls();
+ } else {
+ tv.fill(TimestampData.fromLocalDateTime((LocalDateTime) value));
+ }
+ return tv;
+ case ARRAY:
+ if (value != null) {
+ throw new UnsupportedOperationException("Unsupported create array with default value.");
+ }
+ HeapArrayVector arrayVector = new HeapArrayVector(batchSize);
+ arrayVector.fillWithNulls();
+ return arrayVector;
+ case MAP:
+ case MULTISET:
+ if (value != null) {
+ throw new UnsupportedOperationException(
+ "Unsupported create " + type.getTypeRoot() + " with default value.");
+ }
+ HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null);
+ mapVector.fillWithNulls();
+ return mapVector;
+ case ROW:
+ if (value != null) {
+ throw new UnsupportedOperationException("Unsupported create row with default value.");
+ }
+ RowType rowType = (RowType) type;
+ WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()];
+ for (int i = 0; i < childVectors.length; i++) {
+ childVectors[i] =
+ (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize);
+ }
+ HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors);
+ rowVector.fillWithNulls();
+ return rowVector;
+ case VARIANT:
+ if (value != null) {
+ throw new UnsupportedOperationException("Unsupported create variant with default value.");
+ }
+ HeapRowColumnVector variantVector = new HeapRowColumnVector(
+ batchSize, new HeapBytesVector(batchSize), new HeapBytesVector(batchSize));
+ variantVector.fillWithNulls();
+ return variantVector;
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + type);
+ }
+ }
+
+ private static List<ColumnDescriptor> filterDescriptors(
+ int depth, Type type, List<ColumnDescriptor> columns) throws ParquetRuntimeException {
+ List<ColumnDescriptor> filtered = new ArrayList<>();
+ for (ColumnDescriptor descriptor : columns) {
+ if (depth >= descriptor.getPath().length) {
+ throw new InvalidSchemaException("Expect depth " + depth + " for schema: " + descriptor);
+ }
+ if (type.getName().equals(descriptor.getPath()[depth])) {
+ filtered.add(descriptor);
+ }
+ }
+ ValidationUtils.checkState(filtered.size() > 0, "Corrupted Parquet schema");
+ return filtered;
+ }
+
+ /**
+ * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the
+ * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP},
+ * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by
+ * the supplied pre-built {@link ParquetField} tree.
+ *
+ * @param field the {@link ParquetField} tree for this column, built by
+ * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested
+ * types; ignored for primitives.
+ */
+ public static ColumnReader createColumnReader(
+ boolean utcTimestamp,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> descriptors,
+ PageReadStore pages,
+ @Nullable ParquetField field) throws IOException {
+ switch (fieldType.getTypeRoot()) {
+ case ARRAY:
+ case MAP:
+ case MULTISET:
+ case ROW:
+ case VARIANT:
+ // VARIANT is physically a non-shredded group of two binary leaves (value, metadata), so it
+ // is assembled like a two-field row by the Dremel-style NestedColumnReader (see
+ // NestedColumnReader#readVariant). The ParquetField tree is built by #constructField.
+ Preconditions.checkNotNull(
+ field, "ParquetField must be provided for nested type: %s", fieldType);
+ return new NestedColumnReader(utcTimestamp, pages, field);
+ default:
+ return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages);
+ }
+ }
+
+ /**
+ * Backward-compat entry point kept for callers that don't project nested types and therefore
+ * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload
+ * with a null field; nested types now go through that overload directly.
+ *
+ * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore,
+ * ParquetField)} so nested types take the Dremel path.
+ */
+ @Deprecated
+ public static ColumnReader createColumnReader(
+ boolean utcTimestamp,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> descriptors,
+ PageReadStore pages) throws IOException {
+ return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null);
+ }
+
+ private static ColumnReader createPrimitiveColumnReader(
+ boolean utcTimestamp,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> columns,
+ PageReadStore pages) throws IOException {
+ List<ColumnDescriptor> descriptors = filterDescriptors(0, physicalType, columns);
+ ColumnDescriptor descriptor = descriptors.get(0);
+ PageReader pageReader = pages.getPageReader(descriptor);
+ switch (fieldType.getTypeRoot()) {
+ case BOOLEAN:
+ return new BooleanColumnReader(descriptor, pageReader);
+ case TINYINT:
+ return new ByteColumnReader(descriptor, pageReader);
+ case DOUBLE:
+ return new DoubleColumnReader(descriptor, pageReader);
+ case FLOAT:
+ return new FloatColumnReader(descriptor, pageReader);
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ return new IntColumnReader(descriptor, pageReader);
+ case BIGINT:
+ return new LongColumnReader(descriptor, pageReader);
+ case SMALLINT:
+ return new ShortColumnReader(descriptor, pageReader);
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case BINARY:
+ return new BytesColumnReader(descriptor, pageReader);
+ case FIXED_LEN_BYTE_ARRAY:
+ return new FixedLenBytesColumnReader(descriptor, pageReader);
+ default:
+ throw new AssertionError();
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT64:
+ int precision = fieldType instanceof TimestampType
+ ? ((TimestampType) fieldType).getPrecision()
+ : ((LocalZonedTimestampType) fieldType).getPrecision();
+ return new Int64TimestampColumnReader(utcTimestamp, descriptor, pageReader, precision);
+ case INT96:
+ return new TimestampColumnReader(utcTimestamp, descriptor, pageReader);
+ default:
+ throw new AssertionError(
+ "Unexpected physical type for TIMESTAMP: "
+ + descriptor.getPrimitiveType().getPrimitiveTypeName());
+ }
+ case DECIMAL:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT32:
+ return new IntColumnReader(descriptor, pageReader);
+ case INT64:
+ return new LongColumnReader(descriptor, pageReader);
+ case BINARY:
+ return new BytesColumnReader(descriptor, pageReader);
+ case FIXED_LEN_BYTE_ARRAY:
+ return new FixedLenBytesColumnReader(descriptor, pageReader);
+ default:
+ throw new AssertionError(
+ "Unexpected physical type for DECIMAL: "
+ + descriptor.getPrimitiveType().getPrimitiveTypeName());
+ }
+ default:
+ throw new UnsupportedOperationException(fieldType + " is not supported now.");
+ }
+ }
+
+ /**
+ * Creates the writable column vector that the reader will write into. The returned vector shape
+ * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled
+ * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in
+ * {@link NestedColumnReader} can pass them through unchanged.
+ */
+ public static WritableColumnVector createWritableColumnVector(
+ int batchSize,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> descriptors) {
+ return createWritableColumnVector(batchSize, fieldType, physicalType, descriptors, 0);
+ }
+
+ private static WritableColumnVector createWritableColumnVector(
+ int batchSize,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> columns,
+ int depth) {
+ List<ColumnDescriptor> descriptors = filterDescriptors(depth, physicalType, columns);
+ PrimitiveType primitiveType = descriptors.get(0).getPrimitiveType();
+ PrimitiveType.PrimitiveTypeName typeName = primitiveType.getPrimitiveTypeName();
+ switch (fieldType.getTypeRoot()) {
+ case BOOLEAN:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapBooleanVector(batchSize);
+ case TINYINT:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapByteVector(batchSize);
+ case DOUBLE:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.DOUBLE,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapDoubleVector(batchSize);
+ case FLOAT:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.FLOAT,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapFloatVector(batchSize);
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapIntVector(batchSize);
+ case BIGINT:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.INT64,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapLongVector(batchSize);
+ case SMALLINT:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapShortVector(batchSize);
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ checkArgument(
+ typeName == PrimitiveType.PrimitiveTypeName.BINARY
+ || typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapBytesVector(batchSize);
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ checkArgument(primitiveType.getOriginalType() != OriginalType.TIME_MICROS,
+ getOriginalTypeCheckFailureMessage(primitiveType.getOriginalType(), fieldType));
+ return new HeapTimestampVector(batchSize);
+ case DECIMAL:
+ checkArgument(
+ (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
+ || typeName == PrimitiveType.PrimitiveTypeName.BINARY)
+ && primitiveType.getOriginalType() == OriginalType.DECIMAL,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ return new HeapDecimalVector(batchSize);
+ case ARRAY:
+ ArrayType arrayType = (ArrayType) fieldType;
+ return new HeapArrayVector(
+ batchSize,
+ createWritableColumnVector(
+ batchSize, arrayType.getElementType(), physicalType, descriptors, depth));
+ case MAP: {
+ MapType mapType = (MapType) fieldType;
+ GroupType repeatedType = unwrapMapRepeatedType(physicalType);
+ return new HeapMapColumnVector(
+ batchSize,
+ createWritableColumnVector(
+ batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2),
+ createWritableColumnVector(
+ batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2));
+ }
+ case MULTISET: {
+ MultisetType multisetType = (MultisetType) fieldType;
+ GroupType repeatedType = unwrapMapRepeatedType(physicalType);
+ return new HeapMapColumnVector(
+ batchSize,
+ createWritableColumnVector(
+ batchSize,
+ multisetType.getElementType(),
+ repeatedType.getType(0),
+ descriptors,
+ depth + 2),
+ createWritableColumnVector(
+ batchSize,
+ new IntType(false),
+ repeatedType.getType(1),
+ descriptors,
+ depth + 2));
+ }
+ case ROW:
+ RowType rowType = (RowType) fieldType;
+ GroupType groupType = physicalType.asGroupType();
+ WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()];
+ for (int i = 0; i < columnVectors.length; i++) {
+ int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType);
+ if (fieldIndex < 0) {
+ // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled
+ // vector of the correct shape; NestedColumnReader.readRow will pass it through when the
+ // matching ParquetField child is null.
+ columnVectors[i] =
+ (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize);
+ } else {
+ columnVectors[i] =
+ createWritableColumnVector(
+ batchSize,
+ rowType.getTypeAt(i),
+ groupType.getType(fieldIndex),
+ descriptors,
+ depth + 1);
+ }
+ }
+ return new HeapRowColumnVector(batchSize, columnVectors);
+ case VARIANT:
+ validateVariantType(physicalType);
+ return new HeapRowColumnVector(
+ batchSize,
+ new HeapBytesVector(batchSize),
+ new HeapBytesVector(batchSize));
+ default:
+ throw new UnsupportedOperationException(fieldType + " is not supported now.");
+ }
+ }
+
+ private static void validateVariantType(Type physicalType) {
+ if (!physicalType.isPrimitive()) {
+ GroupType groupType = physicalType.asGroupType();
+ if (isShreddedVariant(groupType)) {
+ throw new UnsupportedOperationException(
+ "Shredded Variant is not supported in Flink. "
+ + "The Parquet group '" + groupType.getName() + "' contains a '"
+ + HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD
+ + "' field indicating a shredded layout.");
+ }
+ validateVariantField(groupType, HoodieSchema.Variant.VARIANT_VALUE_FIELD);
+ validateVariantField(groupType, HoodieSchema.Variant.VARIANT_METADATA_FIELD);
+ } else {
+ throw new IllegalArgumentException(
+ "Type mismatch, expected Variant but got '" + physicalType + "'.");
+ }
+ }
+
+ /**
+ * Checks whether a variant group contains a {@code typed_value} field, indicating a shredded
+ * layout.
+ */
+ private static boolean isShreddedVariant(GroupType groupType) {
+ return groupType.containsField(HoodieSchema.Variant.VARIANT_TYPED_VALUE_FIELD);
+ }
+
+ private static void validateVariantField(GroupType groupType, String fieldName) {
+ if (groupType.containsField(fieldName)) {
+ Type fieldType = groupType.getType(fieldName);
+ if (fieldType.isPrimitive()
+ && fieldType.asPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY) {
+ return;
+ }
+ }
+ throw new IllegalArgumentException(
+ "Invalid Variant Parquet schema: missing binary field '" + fieldName + "'.");
+ }
+
+ /**
+ * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching
+ * Parquet's canonical 3-level map encoding.
+ */
+ private static GroupType unwrapMapRepeatedType(Type physicalType) {
+ return physicalType.asGroupType().getType(0).asGroupType();
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil)
+ //
+ // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children
+ // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This
+ // matches the Hudi schema-evolution contract and is the companion to the null-child branch in
+ // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`.
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed
+ * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally;
+ * primitive top-level fields produce {@code null} entries (callers don't need a tree for those).
+ */
+ public static List<ParquetField> buildFieldsList(
+ List<RowType.RowField> children, List<String> fieldNames, MessageColumnIO columnIO) {
+ List<ParquetField> list = new ArrayList<>();
+ for (int i = 0; i < children.size(); i++) {
+ RowType.RowField child = children.get(i);
+ if (isNestedType(child.getType())) {
+ list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i))));
+ } else {
+ list.add(null);
+ }
+ }
+ return list;
+ }
+
+ private static boolean isNestedType(LogicalType type) {
+ return type instanceof RowType
+ || type instanceof ArrayType
+ || type instanceof MapType
+ || type instanceof MultisetType
+ || type instanceof VariantType;
+ }
+
+ @Nullable
+ private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) {
+ boolean required = columnIO.getType().getRepetition() == REQUIRED;
+ int repetitionLevel = columnIO.getRepetitionLevel();
+ int definitionLevel = columnIO.getDefinitionLevel();
+ LogicalType type = rowField.getType();
+ String fieldName = rowField.getName();
+ if (type instanceof RowType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ RowType rowType = (RowType) type;
+ List<RowType.RowField> childFields = rowType.getFields();
+ List<ParquetField> fieldsList = new ArrayList<>(childFields.size());
+ for (RowType.RowField childField : childFields) {
+ // Hudi schema evolution: a logical child may be absent from the Parquet file. In that
+ // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the
+ // pre-filled null vector instead of recursing.
+ ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName());
+ if (childIo == null) {
+ fieldsList.add(null);
+ } else {
+ fieldsList.add(constructField(childField, childIo));
+ }
+ }
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(fieldsList));
+ }
+
+ if (type instanceof VariantType) {
+ // A variant is physically a non-shredded group of two binary leaves (value, metadata). Build
+ // it as a two-field group of binary primitives so NestedColumnReader#readVariant can assemble
+ // it like a row. Shredded variants (with a typed_value field) are rejected here.
+ validateVariantType(columnIO.getType());
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ LogicalType binaryType = new VarBinaryType(VarBinaryType.MAX_LENGTH);
+ ParquetField valueField =
+ constructField(
+ new RowType.RowField(HoodieSchema.Variant.VARIANT_VALUE_FIELD, binaryType),
+ lookupColumnByName(groupColumnIO, HoodieSchema.Variant.VARIANT_VALUE_FIELD));
+ ParquetField metadataField =
+ constructField(
+ new RowType.RowField(HoodieSchema.Variant.VARIANT_METADATA_FIELD, binaryType),
+ lookupColumnByName(groupColumnIO, HoodieSchema.Variant.VARIANT_METADATA_FIELD));
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(Arrays.asList(valueField, metadataField)));
+ }
+
+ if (type instanceof MapType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
+ MapType mapType = (MapType) type;
+ ParquetField keyField =
+ constructField(
+ new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0));
+ ParquetField valueField =
+ constructField(
+ new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1));
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(Arrays.asList(keyField, valueField)));
+ }
+
+ if (type instanceof MultisetType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
+ MultisetType multisetType = (MultisetType) type;
+ ParquetField keyField =
+ constructField(
+ new RowType.RowField("", multisetType.getElementType()),
+ keyValueColumnIO.getChild(0));
+ ParquetField valueField =
+ constructField(
+ new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1));
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(Arrays.asList(keyField, valueField)));
+ }
+
+ if (type instanceof ArrayType) {
+ ArrayType arrayType = (ArrayType) type;
+ ColumnIO elementTypeColumnIO;
+ if (columnIO instanceof GroupColumnIO) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) {
+ while (!Objects.equals(groupColumnIO.getName(), fieldName)) {
+ groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0);
+ }
+ elementTypeColumnIO = groupColumnIO;
+ } else {
+ if (arrayType.getElementType() instanceof RowType) {
+ elementTypeColumnIO = groupColumnIO;
+ } else {
+ elementTypeColumnIO = groupColumnIO.getChild(0);
+ }
+ }
+ } else if (columnIO instanceof PrimitiveColumnIO) {
+ elementTypeColumnIO = columnIO;
+ } else {
+ throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO));
+ }
+
+ ParquetField elementField =
+ constructField(
+ new RowType.RowField("", arrayType.getElementType()),
+ getArrayElementColumn(elementTypeColumnIO));
+ if (repetitionLevel == elementField.getRepetitionLevel()) {
+ repetitionLevel = columnIO.getParent().getRepetitionLevel();
+ }
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.singletonList(elementField));
+ }
+
+ PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO;
+ return new ParquetPrimitiveField(
+ type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId());
+ }
+
+ /**
+ * Parquet column names are case-insensitive in Flink's lookup. Matches upstream
+ * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent.
+ */
+ public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) {
+ ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName);
+ if (columnIO != null) {
+ return columnIO;
+ }
+ throw new FlinkRuntimeException(
+ "Can not find column io for parquet reader. Column name: " + columnName);
+ }
+
+ /**
+ * Case-insensitive column lookup that returns {@code null} when no match is found — the
+ * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to
+ * emit null {@link ParquetField} children for fields absent from the Parquet file.
+ */
+ @Nullable
+ private static ColumnIO lookupColumnByNameOrNull(
+ GroupColumnIO groupColumnIO, String columnName) {
+ ColumnIO columnIO = groupColumnIO.getChild(columnName);
+ if (columnIO != null) {
+ return columnIO;
+ }
+ for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) {
+ if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) {
+ return groupColumnIO.getChild(i);
+ }
+ }
+ return null;
+ }
+
+ public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) {
+ while (groupColumnIO.getChildrenCount() == 1) {
+ groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0);
+ }
+ return groupColumnIO;
+ }
+
+ public static ColumnIO getArrayElementColumn(ColumnIO columnIO) {
+ while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) {
+ columnIO = ((GroupColumnIO) columnIO).getChild(0);
+ }
+
+ // Three-level list: skip the synthetic `element` / `list` wrapper when present.
+ if (columnIO instanceof GroupColumnIO
+ && columnIO.getType().getLogicalTypeAnnotation() == null
+ && ((GroupColumnIO) columnIO).getChildrenCount() == 1
+ && !columnIO.getName().equals("array")
+ && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) {
+ return ((GroupColumnIO) columnIO).getChild(0);
+ }
+ return columnIO;
+ }
+
+ /**
+ * Returns the field index with given physical row type {@code groupType} and field name
+ * {@code fieldName}.
+ *
+ * @return the physical field index or -1 if the field does not exist
+ */
+ private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) {
+ return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1;
+ }
+
+ private static String getPrimitiveTypeCheckFailureMessage(
+ PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) {
+ return String.format(
+ "Unexpected type exception. Primitive type: %s. Field type: %s.",
+ primitiveType, fieldType.getTypeRoot().name());
+ }
+
+ private static String getOriginalTypeCheckFailureMessage(
+ OriginalType originalType, LogicalType fieldType) {
+ return String.format(
+ "Unexpected type exception. Original type: %s. Field type: %s.",
+ originalType, fieldType.getTypeRoot().name());
+ }
+
+ /**
+ * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience
+ * for callers that need to mirror Hudi's original behaviour where a missing column produces an
+ * explicit null-valued reader rather than being omitted from the batch.
+ */
+ public static ColumnReader<WritableColumnVector> emptyColumnReader() {
+ return new EmptyColumnReader();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
new file mode 100644
index 000000000000..d51d7ee754b8
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+
+/**
+ * Minimal implementation of an array-backed list of booleans.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper.
+ */
+public class BooleanArrayList {
+ private int size;
+ private boolean[] array;
+
+ public BooleanArrayList(int capacity) {
+ this.size = 0;
+ this.array = new boolean[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(boolean element) {
+ grow(size + 1);
+ array[size++] = element;
+ return true;
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return (size == 0);
+ }
+
+ public boolean[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ private void grow(int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final boolean[] t = new boolean[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
new file mode 100644
index 000000000000..4787dbb5b9dd
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+/**
+ * Minimal implementation of an array-backed list of ints.
+ *
+ * <p>Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to
+ * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader.
+ */
+public class IntArrayList {
+
+ private int size;
+ private int[] array;
+
+ public IntArrayList(final int capacity) {
+ this.size = 0;
+ this.array = new int[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(final int number) {
+ grow(size + 1);
+ array[size++] = number;
+ return true;
+ }
+
+ public int removeLast() {
+ if (size == 0) {
+ throw new NoSuchElementException();
+ }
+ --size;
+ return array[size];
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return size == 0;
+ }
+
+ private void grow(final int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final int[] t = new int[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+
+ public int[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ public static final IntArrayList EMPTY =
+ new IntArrayList(0) {
+
+ @Override
+ public boolean add(int number) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int removeLast() {
+ throw new UnsupportedOperationException();
+ }
+ };
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
new file mode 100644
index 000000000000..a51291f9d844
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+
+/**
+ * Minimal implementation of an array-backed list of longs.
+ *
+ * <p>Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to
+ * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader.
+ */
+public class LongArrayList {
+
+ private int size;
+ private long[] array;
+
+ public LongArrayList(int capacity) {
+ this.size = 0;
+ this.array = new long[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(long number) {
+ grow(size + 1);
+ array[size++] = number;
+ return true;
+ }
+
+ public long removeLong(int index) {
+ if (index >= size) {
+ throw new IndexOutOfBoundsException(
+ "Index (" + index + ") is greater than or equal to list size (" + size + ")");
+ }
+ final long old = array[index];
+ size--;
+ if (index != size) {
+ System.arraycopy(array, index + 1, array, index, size - index);
+ }
+ return old;
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return (size == 0);
+ }
+
+ public long[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ private void grow(int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final long[] t = new long[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
new file mode 100644
index 000000000000..3f2f8976b69b
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import org.apache.hudi.table.format.cow.vector.position.CollectionPosition;
+import org.apache.hudi.table.format.cow.vector.position.RowPosition;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+
+import static java.lang.String.format;
+
+/**
+ * Utils to calculate nested type position.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.utils.NestedPositionUtil}).
+ */
+public class NestedPositionUtil {
+
+ /**
+ * Calculate row offsets according to column's max repetition level, definition level, value's
+ * repetition level and definition level. Each row has three situation:
+ * <li>Row is not defined,because it's optional parent fields is null, this is decided by its
+ * parent's repetition level
+ * <li>Row is null
+ * <li>Row is defined and not empty.
+ *
+ * @param field field that contains the row column message include max repetition level and
+ * definition level.
+ * @param fieldRepetitionLevels int array with each value's repetition level.
+ * @param fieldDefinitionLevels int array with each value's definition level.
+ * @return {@link RowPosition} contains collections row count and isNull array.
+ */
+ public static RowPosition calculateRowOffsets(
+ ParquetField field, int[] fieldDefinitionLevels, int[] fieldRepetitionLevels) {
+ int rowDefinitionLevel = field.getDefinitionLevel();
+ int rowRepetitionLevel = field.getRepetitionLevel();
+ int nullValuesCount = 0;
+ BooleanArrayList nullRowFlags = new BooleanArrayList(0);
+ for (int i = 0; i < fieldDefinitionLevels.length; i++) {
+ // If a row's last field is an array, the repetition levels for the array's items will
+ // be larger than the parent row's repetition level, so we need to skip those values.
+ if (fieldRepetitionLevels[i] > rowRepetitionLevel) {
+ continue;
+ }
+
+ if (fieldDefinitionLevels[i] >= rowDefinitionLevel) {
+ // current row is defined and not empty
+ nullRowFlags.add(false);
+ } else {
+ // current row is null
+ nullRowFlags.add(true);
+ nullValuesCount++;
+ }
+ }
+ if (nullValuesCount == 0) {
+ return new RowPosition(null, fieldDefinitionLevels.length);
+ }
+ return new RowPosition(nullRowFlags.toArray(), nullRowFlags.size());
+ }
+
+ /**
+ * Calculate the collection's offsets according to column's max repetition level, definition
+ * level, value's repetition level and definition level. Each collection (Array or Map) has four
+ * situation:
+ * <li>Collection is not defined, because optional parent fields is null, this is decided by its
+ * parent's repetition level
+ * <li>Collection is null
+ * <li>Collection is defined but empty
+ * <li>Collection is defined and not empty. In this case offset value is increased by the number
+ * of elements in that collection
+ *
+ * @param field field that contains array/map column message include max repetition level and
+ * definition level.
+ * @param definitionLevels int array with each value's definition level.
+ * @param repetitionLevels int array with each value's repetition level.
+ * @return {@link CollectionPosition} contains collections offset array, length array and isNull
+ * array.
+ */
+ public static CollectionPosition calculateCollectionOffsets(
+ ParquetField field, int[] definitionLevels, int[] repetitionLevels) {
+ int collectionDefinitionLevel = field.getDefinitionLevel();
+ int collectionRepetitionLevel = field.getRepetitionLevel() + 1;
+ int offset = 0;
+ int valueCount = 0;
+ LongArrayList offsets = new LongArrayList(0);
+ offsets.add(offset);
+ BooleanArrayList emptyCollectionFlags = new BooleanArrayList(0);
+ BooleanArrayList nullCollectionFlags = new BooleanArrayList(0);
+ int nullValuesCount = 0;
+ for (int i = 0;
+ i < definitionLevels.length;
+ i = getNextCollectionStartIndex(repetitionLevels, collectionRepetitionLevel, i)) {
+ valueCount++;
+ if (definitionLevels[i] >= collectionDefinitionLevel - 1) {
+ boolean isNull =
+ isOptionalFieldValueNull(definitionLevels[i], collectionDefinitionLevel);
+ nullCollectionFlags.add(isNull);
+ nullValuesCount += isNull ? 1 : 0;
+ // definitionLevels[i] > collectionDefinitionLevel => Collection is defined and not
+ // empty
+ // definitionLevels[i] == collectionDefinitionLevel => Collection is defined but
+ // empty
+ if (definitionLevels[i] > collectionDefinitionLevel) {
+ emptyCollectionFlags.add(false);
+ offset += getCollectionSize(repetitionLevels, collectionRepetitionLevel, i + 1);
+ } else if (definitionLevels[i] == collectionDefinitionLevel) {
+ offset++;
+ emptyCollectionFlags.add(true);
+ } else {
+ offset++;
+ emptyCollectionFlags.add(false);
+ }
+ offsets.add(offset);
+ } else {
+ // when definitionLevels[i] < collectionDefinitionLevel - 1, it means the collection
+ // is
+ // not defined, but we need to regard it as null to avoid getting value wrong.
+ nullCollectionFlags.add(true);
+ nullValuesCount++;
+ offsets.add(++offset);
+ emptyCollectionFlags.add(false);
+ }
+ }
+ long[] offsetsArray = offsets.toArray();
+ long[] length = calculateLengthByOffsets(emptyCollectionFlags.toArray(), offsetsArray);
+ if (nullValuesCount == 0) {
+ return new CollectionPosition(null, offsetsArray, length, valueCount);
+ }
+ return new CollectionPosition(
+ nullCollectionFlags.toArray(), offsetsArray, length, valueCount);
+ }
+
+ public static boolean isOptionalFieldValueNull(int definitionLevel, int maxDefinitionLevel) {
+ return definitionLevel == maxDefinitionLevel - 1;
+ }
+
+ public static long[] calculateLengthByOffsets(
+ boolean[] collectionIsEmpty, long[] arrayOffsets) {
+ LongArrayList lengthList = new LongArrayList(arrayOffsets.length);
+ for (int i = 0; i < arrayOffsets.length - 1; i++) {
+ long offset = arrayOffsets[i];
+ long length = arrayOffsets[i + 1] - offset;
+ if (length < 0) {
+ throw new IllegalArgumentException(
+ format(
+ "Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s",
+ i, arrayOffsets[i], i + 1, arrayOffsets[i + 1]));
+ }
+ if (collectionIsEmpty[i]) {
+ length = 0;
+ }
+ lengthList.add(length);
+ }
+ return lengthList.toArray();
+ }
+
+ private static int getNextCollectionStartIndex(
+ int[] repetitionLevels, int maxRepetitionLevel, int elementIndex) {
+ do {
+ elementIndex++;
+ } while (hasMoreElements(repetitionLevels, elementIndex)
+ && isNotCollectionBeginningMarker(
+ repetitionLevels, maxRepetitionLevel, elementIndex));
+ return elementIndex;
+ }
+
+ /** This method is only called for non-empty collections. */
+ private static int getCollectionSize(
+ int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) {
+ int size = 1;
+ while (hasMoreElements(repetitionLevels, nextIndex)
+ && isNotCollectionBeginningMarker(
+ repetitionLevels, maxRepetitionLevel, nextIndex)) {
+ // Collection elements cannot only be primitive, but also can have nested structure
+ // Counting only elements which belong to current collection, skipping inner elements of
+ // nested collections/structs
+ if (repetitionLevels[nextIndex] <= maxRepetitionLevel) {
+ size++;
+ }
+ nextIndex++;
+ }
+ return size;
+ }
+
+ private static boolean isNotCollectionBeginningMarker(
+ int[] repetitionLevels, int maxRepetitionLevel, int nextIndex) {
+ return repetitionLevels[nextIndex] >= maxRepetitionLevel;
+ }
+
+ private static boolean hasMoreElements(int[] repetitionLevels, int nextIndex) {
+ return nextIndex < repetitionLevels.length;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
new file mode 100644
index 000000000000..c597bafad1ed
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.columnar.ColumnarArrayData;
+import org.apache.flink.table.data.columnar.vector.ArrayColumnVector;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+
+/**
+ * This class represents a nullable heap array column vector.
+ */
+public class HeapArrayVector extends AbstractHeapVector
+ implements WritableColumnVector, ArrayColumnVector {
+
+ public long[] offsets;
+ public long[] lengths;
+ public ColumnVector child;
+ private int size;
+
+ public HeapArrayVector(int len) {
+ super(len);
+ offsets = new long[len];
+ lengths = new long[len];
+ }
+
+ public HeapArrayVector(int len, ColumnVector vector) {
+ super(len);
+ offsets = new long[len];
+ lengths = new long[len];
+ this.child = vector;
+ }
+
+ public int getLen() {
+ return this.isNull.length;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public void setSize(int size) {
+ this.size = size;
+ }
+
+ // ---------------------------------------------------------------------------------------------
+ // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths}
+ // and {@code child} fields so legacy callers continue to work; the new {@link
+ // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any
+ // future Flink-2.1-style caller use these accessors.
+ // ---------------------------------------------------------------------------------------------
+
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public void setOffsets(long[] offsets) {
+ this.offsets = offsets;
+ }
+
+ public long[] getLengths() {
+ return lengths;
+ }
+
+ public void setLengths(long[] lengths) {
+ this.lengths = lengths;
+ }
+
+ public ColumnVector getChild() {
+ return child;
+ }
+
+ public void setChild(ColumnVector child) {
+ this.child = child;
+ }
+
+ @Override
+ public ArrayData getArray(int i) {
+ long offset = offsets[i];
+ long length = lengths[i];
+ return new ColumnarArrayData(child, (int) offset, (int) length);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java
new file mode 100644
index 000000000000..fdc55ac18fc6
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.columnar.vector.DecimalColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+
+/**
+ * This class represents a nullable heap map decimal vector.
+ */
+public class HeapDecimalVector extends HeapBytesVector implements DecimalColumnVector {
+
+ public HeapDecimalVector(int len) {
+ super(len);
+ }
+
+ @Override
+ public DecimalData getDecimal(int i, int precision, int scale) {
+ return DecimalData.fromUnscaledBytes(
+ this.getBytes(i).getBytes(), precision, scale);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
new file mode 100644
index 000000000000..293604e02c4a
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.columnar.ColumnarMapData;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.MapColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+
+/**
+ * This class represents a nullable heap map column vector.
+ *
+ * <p>Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from
+ * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward
+ * compatibility: the {@code keys} / {@code values} fields are typed
+ * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing
+ * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original
+ * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use
+ * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}.
+ */
+public class HeapMapColumnVector extends AbstractHeapVector
+ implements WritableColumnVector, MapColumnVector {
+
+ private WritableColumnVector keys;
+ private WritableColumnVector values;
+
+ // ---------------------------------------------------------------------------------------------
+ // Flink 2.1 Dremel-style state. Populated by {@link
+ // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and
+ // consumed by {@link #getMap(int)}.
+ // ---------------------------------------------------------------------------------------------
+ private long[] offsets;
+ private long[] lengths;
+ private int size;
+
+ public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) {
+ super(len);
+ this.offsets = new long[len];
+ this.lengths = new long[len];
+ this.keys = keys;
+ this.values = values;
+ }
+
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public void setOffsets(long[] offsets) {
+ this.offsets = offsets;
+ }
+
+ public long[] getLengths() {
+ return lengths;
+ }
+
+ public void setLengths(long[] lengths) {
+ this.lengths = lengths;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public void setSize(int size) {
+ this.size = size;
+ }
+
+ public WritableColumnVector getKeys() {
+ return keys;
+ }
+
+ public void setKeys(WritableColumnVector keys) {
+ this.keys = keys;
+ }
+
+ public WritableColumnVector getValues() {
+ return values;
+ }
+
+ public void setValues(WritableColumnVector values) {
+ this.values = values;
+ }
+
+ /**
+ * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract
+ * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}.
+ */
+ public ColumnVector getKeyColumnVector() {
+ return keys;
+ }
+
+ /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */
+ public ColumnVector getValueColumnVector() {
+ return values;
+ }
+
+ @Override
+ public MapData getMap(int rowId) {
+ long offset = offsets[rowId];
+ long length = lengths[rowId];
+ return new ColumnarMapData(keys, values, (int) offset, (int) length);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
new file mode 100644
index 000000000000..0c640ce92ee4
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.columnar.ColumnarRowData;
+import org.apache.flink.table.data.columnar.vector.RowColumnVector;
+import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch;
+import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+
+/**
+ * This class represents a nullable heap row column vector.
+ */
+public class HeapRowColumnVector extends AbstractHeapVector
+ implements WritableColumnVector, RowColumnVector {
+
+ public WritableColumnVector[] vectors;
+
+ public HeapRowColumnVector(int len, WritableColumnVector... vectors) {
+ super(len);
+ this.vectors = vectors;
+ }
+
+ /**
+ * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code
+ * vectors} field so legacy callers continue to work; the new {@link
+ * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any
+ * future Flink-2.1-style caller use this accessor.
+ */
+ public WritableColumnVector[] getFields() {
+ return vectors;
+ }
+
+ /** Counterpart of {@link #getFields()}. */
+ public void setFields(WritableColumnVector[] fields) {
+ this.vectors = fields;
+ }
+
+ @Override
+ public ColumnarRowData getRow(int i) {
+ ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors));
+ columnarRowData.setRowId(i);
+ return columnarRowData;
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ for (WritableColumnVector vector : vectors) {
+ vector.reset();
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
new file mode 100644
index 000000000000..a37b88352cf5
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.columnar.vector.BytesColumnVector;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.DecimalColumnVector;
+import org.apache.flink.table.data.columnar.vector.Dictionary;
+import org.apache.flink.table.data.columnar.vector.IntColumnVector;
+import org.apache.flink.table.data.columnar.vector.LongColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide
+ * {@link DecimalColumnVector} interface.
+ */
+public class ParquetDecimalVector
+ implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector {
+
+ private final ColumnVector vector;
+
+ public ParquetDecimalVector(ColumnVector vector) {
+ this.vector = vector;
+ }
+
+ @Override
+ public DecimalData getDecimal(int i, int precision, int scale) {
+ if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) {
+ return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale);
+ } else if (ParquetSchemaConverter.is64BitDecimal(precision)
+ && vector instanceof LongColumnVector) {
+ return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale);
+ } else {
+ checkArgument(
+ vector instanceof BytesColumnVector,
+ "Reading decimal type occur unsupported vector type: %s",
+ vector.getClass());
+ return DecimalData.fromUnscaledBytes(
+ ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale);
+ }
+ }
+
+ public ColumnVector getVector() {
+ return vector;
+ }
+
+ @Override
+ public boolean isNullAt(int i) {
+ return vector.isNullAt(i);
+ }
+
+ @Override
+ public void reset() {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).reset();
+ }
+ }
+
+ @Override
+ public void setNullAt(int rowId) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setNullAt(rowId);
+ }
+ }
+
+ @Override
+ public void setNulls(int rowId, int count) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setNulls(rowId, count);
+ }
+ }
+
+ @Override
+ public void fillWithNulls() {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).fillWithNulls();
+ }
+ }
+
+ @Override
+ public void setDictionary(Dictionary dictionary) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setDictionary(dictionary);
+ }
+ }
+
+ @Override
+ public boolean hasDictionary() {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).hasDictionary();
+ }
+ return false;
+ }
+
+ @Override
+ public WritableIntVector reserveDictionaryIds(int capacity) {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).reserveDictionaryIds(capacity);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public WritableIntVector getDictionaryIds() {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).getDictionaryIds();
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public Bytes getBytes(int i) {
+ if (vector instanceof WritableBytesVector) {
+ return ((WritableBytesVector) vector).getBytes(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void appendBytes(int rowId, byte[] value, int offset, int length) {
+ if (vector instanceof WritableBytesVector) {
+ ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length);
+ }
+ }
+
+ @Override
+ public void fill(byte[] value) {
+ if (vector instanceof WritableBytesVector) {
+ ((WritableBytesVector) vector).fill(value);
+ }
+ }
+
+ @Override
+ public int getInt(int i) {
+ if (vector instanceof WritableIntVector) {
+ return ((WritableIntVector) vector).getInt(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void setInt(int rowId, int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInt(rowId, value);
+ }
+ }
+
+ @Override
+ public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void setInts(int rowId, int count, int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInts(rowId, count, value);
+ }
+ }
+
+ @Override
+ public void setInts(int rowId, int count, int[] src, int srcIndex) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void fill(int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).fill(value);
+ }
+ }
+
+ @Override
+ public long getLong(int i) {
+ if (vector instanceof WritableLongVector) {
+ return ((WritableLongVector) vector).getLong(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void setLong(int rowId, long value) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).setLong(rowId, value);
+ }
+ }
+
+ @Override
+ public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void fill(long value) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).fill(value);
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
new file mode 100644
index 000000000000..fcdedfbc9d71
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+import javax.annotation.Nullable;
+
+/**
+ * To represent collection's position in repeated type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.CollectionPosition}).
+ */
+public class CollectionPosition {
+ @Nullable private final boolean[] isNull;
+ private final long[] offsets;
+ private final long[] length;
+ private final int valueCount;
+
+ public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) {
+ this.isNull = isNull;
+ this.offsets = offsets;
+ this.length = length;
+ this.valueCount = valueCount;
+ }
+
+ public boolean[] getIsNull() {
+ return isNull;
+ }
+
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public long[] getLength() {
+ return length;
+ }
+
+ public int getValueCount() {
+ return valueCount;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
new file mode 100644
index 000000000000..fe95419ac321
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+/**
+ * To delegate repetition level and definition level.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.LevelDelegation}).
+ */
+public class LevelDelegation {
+ private final int[] repetitionLevel;
+ private final int[] definitionLevel;
+
+ public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) {
+ this.repetitionLevel = repetitionLevel;
+ this.definitionLevel = definitionLevel;
+ }
+
+ public int[] getRepetitionLevel() {
+ return repetitionLevel;
+ }
+
+ public int[] getDefinitionLevel() {
+ return definitionLevel;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
new file mode 100644
index 000000000000..5438b6797323
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+import javax.annotation.Nullable;
+
+/**
+ * To represent struct's position in repeated type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.RowPosition}).
+ */
+public class RowPosition {
+ @Nullable private final boolean[] isNull;
+ private final int positionsCount;
+
+ public RowPosition(boolean[] isNull, int positionsCount) {
+ this.isNull = isNull;
+ this.positionsCount = positionsCount;
+ }
+
+ public boolean[] getIsNull() {
+ return isNull;
+ }
+
+ public int getPositionsCount() {
+ return positionsCount;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
new file mode 100644
index 000000000000..9fbec664c599
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.formats.parquet.vector.ParquetDictionary;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.DataPageV1;
+import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.schema.PrimitiveType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
+
+/**
+ * Abstract {@link ColumnReader}.
+ * See {@link org.apache.parquet.column.impl.ColumnReaderImpl},
+ * part of the code is referred from Apache Spark and Apache Parquet.
+ *
+ * <p>Note: Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.reader.AbstractColumnReader}
+ * because some of the package scope methods.
+ */
+@Slf4j(topic = "org.apache.flink.formats.parquet.vector.reader.AbstractColumnReader")
+public abstract class AbstractColumnReader<V extends WritableColumnVector>
+ implements ColumnReader<V> {
+
+ private final PageReader pageReader;
+
+ /**
+ * The dictionary, if this column has dictionary encoding.
+ */
+ protected final Dictionary dictionary;
+
+ /**
+ * Maximum definition level for this column.
+ */
+ protected final int maxDefLevel;
+
+ protected final ColumnDescriptor descriptor;
+
+ /**
+ * Total number of values read.
+ */
+ private long valuesRead;
+
+ /**
+ * value that indicates the end of the current page. That is, if valuesRead ==
+ * endOfPageValueCount, we are at the end of the page.
+ */
+ private long endOfPageValueCount;
+
+ /**
+ * If true, the current page is dictionary encoded.
+ */
+ private boolean isCurrentPageDictionaryEncoded;
+
+ /**
+ * Total values in the current page.
+ */
+ private int pageValueCount;
+
+ /*
+ * Input streams:
+ * 1.Run length encoder to encode every data, so we have run length stream to get
+ * run length information.
+ * 2.Data maybe is real data, maybe is dictionary ids which need be decoded to real
+ * data from Dictionary.
+ *
+ * Run length stream ------> Data stream
+ * |
+ * ------> Dictionary ids stream
+ */
+
+ /**
+ * Run length decoder for data and dictionary.
+ */
+ protected RunLengthDecoder runLenDecoder;
+
+ /**
+ * Data input stream.
+ */
+ ByteBufferInputStream dataInputStream;
+
+ /**
+ * Dictionary decoder to wrap dictionary ids input stream.
+ */
+ private RunLengthDecoder dictionaryIdsDecoder;
+
+ public AbstractColumnReader(
+ ColumnDescriptor descriptor,
+ PageReader pageReader) throws IOException {
+ this.descriptor = descriptor;
+ this.pageReader = pageReader;
+ this.maxDefLevel = descriptor.getMaxDefinitionLevel();
+
+ DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
+ if (dictionaryPage != null) {
+ try {
+ this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
+ this.isCurrentPageDictionaryEncoded = true;
+ } catch (IOException e) {
+ throw new IOException("could not decode the dictionary for " + descriptor, e);
+ }
+ } else {
+ this.dictionary = null;
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+ /*
+ * Total number of values in this column (in this row group).
+ */
+ long totalValueCount = pageReader.getTotalValueCount();
+ if (totalValueCount == 0) {
+ throw new IOException("totalValueCount == 0");
+ }
+ }
+
+ protected void checkTypeName(PrimitiveType.PrimitiveTypeName expectedName) {
+ PrimitiveType.PrimitiveTypeName actualName = descriptor.getPrimitiveType().getPrimitiveTypeName();
+ Preconditions.checkArgument(
+ actualName == expectedName,
+ "Expected type name: %s, actual type name: %s",
+ expectedName,
+ actualName);
+ }
+
+ /**
+ * Reads `total` values from this columnReader into column.
+ */
+ @Override
+ public final void readToVector(int readNumber, V vector) throws IOException {
+ int rowId = 0;
+ WritableIntVector dictionaryIds = null;
+ if (dictionary != null) {
+ dictionaryIds = vector.reserveDictionaryIds(readNumber);
+ }
+ while (readNumber > 0) {
+ // Compute the number of values we want to read in this page.
+ int leftInPage = (int) (endOfPageValueCount - valuesRead);
+ if (leftInPage == 0) {
+ DataPage page = pageReader.readPage();
+ if (page instanceof DataPageV1) {
+ readPageV1((DataPageV1) page);
+ } else if (page instanceof DataPageV2) {
+ readPageV2((DataPageV2) page);
+ } else {
+ throw new RuntimeException("Unsupported page type: " + page.getClass());
+ }
+ leftInPage = (int) (endOfPageValueCount - valuesRead);
+ }
+ int num = Math.min(readNumber, leftInPage);
+ if (isCurrentPageDictionaryEncoded) {
+ // Read and decode dictionary ids.
+ runLenDecoder.readDictionaryIds(
+ num, dictionaryIds, vector, rowId, maxDefLevel, this.dictionaryIdsDecoder);
+
+ if (vector.hasDictionary() || (rowId == 0 && supportLazyDecode())) {
+ // Column vector supports lazy decoding of dictionary values so just set the dictionary.
+ // We can't do this if rowId != 0 AND the column doesn't have a dictionary (i.e. some
+ // non-dictionary encoded values have already been added).
+ vector.setDictionary(new ParquetDictionary(dictionary, descriptor));
+ } else {
+ readBatchFromDictionaryIds(rowId, num, vector, dictionaryIds);
+ }
+ } else {
+ if (vector.hasDictionary() && rowId != 0) {
+ // This batch already has dictionary encoded values but this new page is not. The batch
+ // does not support a mix of dictionary and not so we will decode the dictionary.
+ readBatchFromDictionaryIds(0, rowId, vector, vector.getDictionaryIds());
+ }
+ vector.setDictionary(null);
+ readBatch(rowId, num, vector);
+ }
+
+ valuesRead += num;
+ rowId += num;
+ readNumber -= num;
+ }
+ }
+
+ private void readPageV1(DataPageV1 page) throws IOException {
+ this.pageValueCount = page.getValueCount();
+ ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
+
+ // Initialize the decoders.
+ if (page.getDlEncoding() != Encoding.RLE && descriptor.getMaxDefinitionLevel() != 0) {
+ throw new UnsupportedOperationException("Unsupported encoding: " + page.getDlEncoding());
+ }
+ int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
+ this.runLenDecoder = new RunLengthDecoder(bitWidth);
+ try {
+ BytesInput bytes = page.getBytes();
+ ByteBufferInputStream in = bytes.toInputStream();
+ rlReader.initFromPage(pageValueCount, in);
+ this.runLenDecoder.initFromStream(pageValueCount, in);
+ prepareNewPage(page.getValueEncoding(), in);
+ } catch (IOException e) {
+ throw new IOException("could not read page " + page + " in col " + descriptor, e);
+ }
+ }
+
+ private void readPageV2(DataPageV2 page) throws IOException {
+ this.pageValueCount = page.getValueCount();
+
+ int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
+ // do not read the length from the stream. v2 pages handle dividing the page bytes.
+ this.runLenDecoder = new RunLengthDecoder(bitWidth, false);
+ this.runLenDecoder.initFromStream(
+ this.pageValueCount, page.getDefinitionLevels().toInputStream());
+ try {
+ prepareNewPage(page.getDataEncoding(), page.getData().toInputStream());
+ } catch (IOException e) {
+ throw new IOException("could not read page " + page + " in col " + descriptor, e);
+ }
+ }
+
+ private void prepareNewPage(
+ Encoding dataEncoding,
+ ByteBufferInputStream in) throws IOException {
+ this.endOfPageValueCount = valuesRead + pageValueCount;
+ if (dataEncoding.usesDictionary()) {
+ if (dictionary == null) {
+ throw new IOException("Could not read page in col "
+ + descriptor
+ + " as the dictionary was missing for encoding "
+ + dataEncoding);
+ }
+ @SuppressWarnings("deprecation")
+ Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
+ if (dataEncoding != plainDict && dataEncoding != Encoding.RLE_DICTIONARY) {
+ throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
+ }
+ this.dataInputStream = null;
+ this.dictionaryIdsDecoder = new RunLengthDecoder();
+ try {
+ this.dictionaryIdsDecoder.initFromStream(pageValueCount, in);
+ } catch (IOException e) {
+ throw new IOException("could not read dictionary in col " + descriptor, e);
+ }
+ this.isCurrentPageDictionaryEncoded = true;
+ } else {
+ if (dataEncoding != Encoding.PLAIN) {
+ throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
+ }
+ this.dictionaryIdsDecoder = null;
+ log.debug("init from page at offset {} for length {}", in.position(), in.available());
+ this.dataInputStream = in.remainingStream();
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+
+ afterReadPage();
+ }
+
+ final ByteBuffer readDataBuffer(int length) {
+ try {
+ return dataInputStream.slice(length).order(ByteOrder.LITTLE_ENDIAN);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("Failed to read " + length + " bytes", e);
+ }
+ }
+
+ /**
+ * After read a page, we may need some initialization.
+ */
+ protected void afterReadPage() {
+ }
+
+ /**
+ * Support lazy dictionary ids decode. See more in {@link ParquetDictionary}.
+ * If return false, we will decode all the data first.
+ */
+ protected boolean supportLazyDecode() {
+ return true;
+ }
+
+ /**
+ * Read batch from {@link #runLenDecoder} and {@link #dataInputStream}.
+ */
+ protected abstract void readBatch(int rowId, int num, V column);
+
+ /**
+ * Decode dictionary ids to data.
+ * From {@link #runLenDecoder} and {@link #dictionaryIdsDecoder}.
+ */
+ protected abstract void readBatchFromDictionaryIds(
+ int rowId,
+ int num,
+ V column,
+ WritableIntVector dictionaryIds);
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
new file mode 100644
index 000000000000..d743660852a6
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.DataPageV1;
+import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.schema.Type;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.VALUES;
+
+/**
+ * Abstract {@link ColumnReader}. part of the code is referred from Apache Hive and Apache Parquet.
+ */
+@Slf4j
+public abstract class BaseVectorizedColumnReader implements ColumnReader<WritableColumnVector> {
+
+ protected boolean isUtcTimestamp;
+
+ /**
+ * Total number of values read.
+ */
+ protected long valuesRead;
+
+ /**
+ * value that indicates the end of the current page. That is, if valuesRead ==
+ * endOfPageValueCount, we are at the end of the page.
+ */
+ protected long endOfPageValueCount;
+
+ /**
+ * The dictionary, if this column has dictionary encoding.
+ */
+ protected final ParquetDataColumnReader dictionary;
+
+ /**
+ * If true, the current page is dictionary encoded.
+ */
+ protected boolean isCurrentPageDictionaryEncoded;
+
+ /**
+ * Maximum definition level for this column.
+ */
+ protected final int maxDefLevel;
+
+ protected int definitionLevel;
+ protected int repetitionLevel;
+
+ /**
+ * Repetition/Definition/Value readers.
+ */
+ protected IntIterator repetitionLevelColumn;
+
+ protected IntIterator definitionLevelColumn;
+ protected ParquetDataColumnReader dataColumn;
+
+ /**
+ * Total values in the current page.
+ */
+ protected int pageValueCount;
+
+ protected final PageReader pageReader;
+ protected final ColumnDescriptor descriptor;
+ protected final Type type;
+ protected final LogicalType logicalType;
+
+ public BaseVectorizedColumnReader(
+ ColumnDescriptor descriptor,
+ PageReader pageReader,
+ boolean isUtcTimestamp,
+ Type parquetType,
+ LogicalType logicalType)
+ throws IOException {
+ this.descriptor = descriptor;
+ this.type = parquetType;
+ this.pageReader = pageReader;
+ this.maxDefLevel = descriptor.getMaxDefinitionLevel();
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.logicalType = logicalType;
+
+ DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
+ if (dictionaryPage != null) {
+ try {
+ this.dictionary =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ parquetType.asPrimitiveType(),
+ dictionaryPage
+ .getEncoding()
+ .initDictionary(descriptor, dictionaryPage),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } catch (IOException e) {
+ throw new IOException("could not decode the dictionary for " + descriptor, e);
+ }
+ } else {
+ this.dictionary = null;
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+ }
+
+ protected void readRepetitionAndDefinitionLevels() {
+ repetitionLevel = repetitionLevelColumn.nextInt();
+ definitionLevel = definitionLevelColumn.nextInt();
+ valuesRead++;
+ }
+
+ protected void readPage() throws IOException {
+ DataPage page = pageReader.readPage();
+
+ if (page == null) {
+ return;
+ }
+
+ page.accept(
+ new DataPage.Visitor<Void>() {
+ @Override
+ public Void visit(DataPageV1 dataPageV1) {
+ readPageV1(dataPageV1);
+ return null;
+ }
+
+ @Override
+ public Void visit(DataPageV2 dataPageV2) {
+ readPageV2(dataPageV2);
+ return null;
+ }
+ });
+ }
+
+ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount)
+ throws IOException {
+ this.pageValueCount = valueCount;
+ this.endOfPageValueCount = valuesRead + pageValueCount;
+ if (dataEncoding.usesDictionary()) {
+ this.dataColumn = null;
+ if (dictionary == null) {
+ throw new IOException(
+ "could not read page in col "
+ + descriptor
+ + " as the dictionary was missing for encoding "
+ + dataEncoding);
+ }
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getDictionaryBasedValuesReader(
+ descriptor, VALUES, dictionary.getDictionary()),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } else {
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getValuesReader(descriptor, VALUES),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+
+ try {
+ dataColumn.initFromPage(pageValueCount, in);
+ } catch (IOException e) {
+ throw new IOException("could not read page in col " + descriptor, e);
+ }
+ }
+
+ private void readPageV1(DataPageV1 page) {
+ ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
+ ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL);
+ this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
+ this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
+ try {
+ BytesInput bytes = page.getBytes();
+ log.debug("page size {} bytes and {} records", bytes.size(), pageValueCount);
+ ByteBufferInputStream in = bytes.toInputStream();
+ log.debug("reading repetition levels at {}", in.position());
+ rlReader.initFromPage(pageValueCount, in);
+ log.debug("reading definition levels at {}", in.position());
+ dlReader.initFromPage(pageValueCount, in);
+ log.debug("reading data at {}", in.position());
+ initDataReader(page.getValueEncoding(), in, page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ "could not read page " + page + " in col " + descriptor, e);
+ }
+ }
+
+ private void readPageV2(DataPageV2 page) {
+ this.pageValueCount = page.getValueCount();
+ this.repetitionLevelColumn =
+ newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels());
+ this.definitionLevelColumn =
+ newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels());
+ try {
+ log.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount);
+ initDataReader(
+ page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ "could not read page " + page + " in col " + descriptor, e);
+ }
+ }
+
+ private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
+ try {
+ if (maxLevel == 0) {
+ return new NullIntIterator();
+ }
+ return new RLEIntIterator(
+ new RunLengthBitPackingHybridDecoder(
+ BytesUtils.getWidthFromMaxInt(maxLevel),
+ new ByteArrayInputStream(bytes.toByteArray())));
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ "could not read levels in page for col " + descriptor, e);
+ }
+ }
+
+ /**
+ * Utility classes to abstract over different way to read ints with different encodings.
+ */
+ abstract static class IntIterator {
+ abstract int nextInt();
+ }
+
+ /**
+ * read ints from {@link ValuesReader}.
+ */
+ protected static final class ValuesReaderIntIterator extends IntIterator {
+ ValuesReader delegate;
+
+ public ValuesReaderIntIterator(ValuesReader delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ int nextInt() {
+ return delegate.readInteger();
+ }
+ }
+
+ /**
+ * read ints from {@link RunLengthBitPackingHybridDecoder}.
+ */
+ protected static final class RLEIntIterator extends IntIterator {
+ RunLengthBitPackingHybridDecoder delegate;
+
+ public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ int nextInt() {
+ try {
+ return delegate.readInt();
+ } catch (IOException e) {
+ throw new ParquetDecodingException(e);
+ }
+ }
+ }
+
+ /**
+ * return zero.
+ */
+ protected static final class NullIntIterator extends IntIterator {
+ @Override
+ int nextInt() {
+ return 0;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
new file mode 100644
index 000000000000..a8e33d8c6f7e
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+
+import java.io.IOException;
+
+/**
+ * Empty {@link ColumnReader}.
+ * <p>
+ * This reader is to handle parquet files that have not been updated to the latest Schema.
+ * When reading a parquet file with the latest schema, parquet file might not have the new field.
+ * The EmptyColumnReader is used to handle such scenarios.
+ */
+public class EmptyColumnReader implements ColumnReader<WritableColumnVector> {
+
+ public EmptyColumnReader() {
+ }
+
+ @Override
+ public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
+ vector.fillWithNulls();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
new file mode 100644
index 000000000000..be50e6c6239d
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Fixed length bytes {@code ColumnReader}, just for decimal.
+ *
+ * <p>Note: Reference Flink release 1.13.2
+ * {@code org.apache.flink.formats.parquet.vector.reader.FixedLenBytesColumnReader}
+ * to always write as legacy decimal format.
+ */
+public class FixedLenBytesColumnReader<V extends WritableColumnVector>
+ extends AbstractColumnReader<V> {
+
+ public FixedLenBytesColumnReader(
+ ColumnDescriptor descriptor, PageReader pageReader) throws IOException {
+ super(descriptor, pageReader);
+ checkTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY);
+ }
+
+ @Override
+ protected void readBatch(int rowId, int num, V column) {
+ int bytesLen = descriptor.getPrimitiveType().getTypeLength();
+ WritableBytesVector bytesVector = (WritableBytesVector) column;
+ for (int i = 0; i < num; i++) {
+ if (runLenDecoder.readInteger() == maxDefLevel) {
+ byte[] bytes = readDataBinary(bytesLen).getBytes();
+ bytesVector.appendBytes(rowId + i, bytes, 0, bytes.length);
+ } else {
+ bytesVector.setNullAt(rowId + i);
+ }
+ }
+ }
+
+ @Override
+ protected void readBatchFromDictionaryIds(
+ int rowId, int num, V column, WritableIntVector dictionaryIds) {
+ WritableBytesVector bytesVector = (WritableBytesVector) column;
+ for (int i = rowId; i < rowId + num; ++i) {
+ if (!bytesVector.isNullAt(i)) {
+ byte[] v = dictionary.decodeToBinary(dictionaryIds.getInt(i)).getBytes();
+ bytesVector.appendBytes(i, v, 0, v.length);
+ }
+ }
+ }
+
+ private Binary readDataBinary(int len) {
+ ByteBuffer buffer = readDataBuffer(len);
+ if (buffer.hasArray()) {
+ return Binary.fromConstantByteArray(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), len);
+ } else {
+ byte[] bytes = new byte[len];
+ buffer.get(bytes);
+ return Binary.fromConstantByteArray(bytes);
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
new file mode 100644
index 000000000000..b44273b57ca2
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableTimestampVector;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.schema.PrimitiveType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * Timestamp {@link org.apache.flink.formats.parquet.vector.reader.ColumnReader} that supports INT64 8 bytes,
+ * TIMESTAMP_MILLIS is the deprecated ConvertedType counterpart of a TIMESTAMP logical type
+ * that is UTC normalized and has MILLIS precision.
+ *
+ * <p>See https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp
+ * TIMESTAMP_MILLIS and TIMESTAMP_MICROS are the deprecated ConvertedType.
+ */
+public class Int64TimestampColumnReader extends AbstractColumnReader<WritableTimestampVector> {
+
+ private final boolean utcTimestamp;
+
+ private final ChronoUnit chronoUnit;
+
+ public Int64TimestampColumnReader(
+ boolean utcTimestamp,
+ ColumnDescriptor descriptor,
+ PageReader pageReader,
+ int precision) throws IOException {
+ super(descriptor, pageReader);
+ this.utcTimestamp = utcTimestamp;
+ if (precision <= 3) {
+ this.chronoUnit = ChronoUnit.MILLIS;
+ } else if (precision <= 6) {
+ this.chronoUnit = ChronoUnit.MICROS;
+ } else {
+ throw new IllegalArgumentException(
+ "Avro does not support TIMESTAMP type with precision: "
+ + precision
+ + ", it only support precisions <= 6.");
+ }
+ checkTypeName(PrimitiveType.PrimitiveTypeName.INT64);
+ }
+
+ @Override
+ protected boolean supportLazyDecode() {
+ return false;
+ }
+
+ @Override
+ protected void readBatch(int rowId, int num, WritableTimestampVector column) {
+ for (int i = 0; i < num; i++) {
+ if (runLenDecoder.readInteger() == maxDefLevel) {
+ ByteBuffer buffer = readDataBuffer(8);
+ column.setTimestamp(rowId + i, int64ToTimestamp(utcTimestamp, buffer.getLong(), chronoUnit));
+ } else {
+ column.setNullAt(rowId + i);
+ }
+ }
+ }
+
+ @Override
+ protected void readBatchFromDictionaryIds(
+ int rowId,
+ int num,
+ WritableTimestampVector column,
+ WritableIntVector dictionaryIds) {
+ for (int i = rowId; i < rowId + num; ++i) {
+ if (!column.isNullAt(i)) {
+ column.setTimestamp(i, decodeInt64ToTimestamp(
+ utcTimestamp, dictionary, dictionaryIds.getInt(i), chronoUnit));
+ }
+ }
+ }
+
+ public static TimestampData decodeInt64ToTimestamp(
+ boolean utcTimestamp,
+ org.apache.parquet.column.Dictionary dictionary,
+ int id,
+ ChronoUnit unit) {
+ long value = dictionary.decodeToLong(id);
+ return int64ToTimestamp(utcTimestamp, value, unit);
+ }
+
+ private static TimestampData int64ToTimestamp(
+ boolean utcTimestamp,
+ long interval,
+ ChronoUnit unit) {
+ final Instant instant = Instant.EPOCH.plus(interval, unit);
+ if (utcTimestamp) {
+ return TimestampData.fromInstant(instant);
+ } else {
+ // this applies the local timezone
+ return TimestampData.fromTimestamp(Timestamp.from(instant));
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
new file mode 100644
index 000000000000..7be03dacc218
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.hudi.table.format.cow.utils.NestedPositionUtil;
+import org.apache.hudi.table.format.cow.vector.HeapArrayVector;
+import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector;
+import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector;
+import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
+import org.apache.hudi.table.format.cow.vector.position.CollectionPosition;
+import org.apache.hudi.table.format.cow.vector.position.LevelDelegation;
+import org.apache.hudi.table.format.cow.vector.position.RowPosition;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VariantType;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.Preconditions;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReadStore;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code
+ * Row}). Resolves nested structures using Dremel striping/assembly; see <a
+ * href="https://github.com/julienledem/redelm/wiki/The-striping-and-assembly-algorithms-from-the-Dremel-paper">the
+ * striping and assembly algorithms from the Dremel paper</a>.
+ *
+ * <p>Vendored from Apache Flink 2.1 (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream:
+ *
+ * <ul>
+ * <li>Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code
+ * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code
+ * HeapMapVector}/{@code HeapArrayVector}.
+ * <li>Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a
+ * {@link RowType} may contain {@code null} children — meaning the corresponding logical
+ * field is absent from the Parquet file. Those slots are passed through unchanged and do
+ * not contribute to the row's repetition/definition-level stream.
+ * </ul>
+ */
+public class NestedColumnReader implements ColumnReader<WritableColumnVector> {
+
+ private final Map<ColumnDescriptor, NestedPrimitiveColumnReader> columnReaders;
+ private final boolean isUtcTimestamp;
+
+ private final PageReadStore pages;
+
+ private final ParquetField field;
+
+ public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) {
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.pages = pages;
+ this.field = field;
+ this.columnReaders = new HashMap<>();
+ }
+
+ @Override
+ public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
+ readData(field, readNumber, vector, false);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readData(
+ ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException {
+ if (field.getType() instanceof RowType) {
+ return readRow((ParquetGroupField) field, readNumber, vector, inside);
+ } else if (field.getType() instanceof VariantType) {
+ return readRow((ParquetGroupField) field, readNumber, vector, inside);
+ } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) {
+ return readMap((ParquetGroupField) field, readNumber, vector, inside);
+ } else if (field.getType() instanceof ArrayType) {
+ return readArray((ParquetGroupField) field, readNumber, vector, inside);
+ } else {
+ return readPrimitive((ParquetPrimitiveField) field, readNumber, vector);
+ }
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readRow(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector;
+ LevelDelegation levelDelegation = null;
+ List<ParquetField> children = field.getChildren();
+ WritableColumnVector[] childrenVectors = heapRowVector.getFields();
+ WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length];
+ for (int i = 0; i < children.size(); i++) {
+ ParquetField child = children.get(i);
+ if (child == null) {
+ // Schema-evolution: the logical field is not present in the Parquet file. The slot
+ // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector
+ // (ROW branch), but HeapRowColumnVector#reset() (invoked once per batch by
+ // ParquetColumnarRowSplitReader#nextBatch) cascades to the children and clears those null
+ // flags. Since an absent field is never re-read, re-apply the nulls here so the column stays
+ // NULL instead of reverting to the type's zero value. Skip contributing to the level stream.
+ childrenVectors[i].fillWithNulls();
+ finalChildrenVectors[i] = childrenVectors[i];
+ continue;
+ }
+ Tuple2<LevelDelegation, WritableColumnVector> tuple =
+ readData(child, readNumber, childrenVectors[i], true);
+ levelDelegation = tuple.f0;
+ finalChildrenVectors[i] = tuple.f1;
+ }
+ if (levelDelegation == null) {
+ throw new FlinkRuntimeException(
+ String.format("Row field does not have any non-null children: %s.", field));
+ }
+
+ RowPosition rowPosition =
+ NestedPositionUtil.calculateRowOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If row was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors);
+ } else {
+ heapRowVector.setFields(finalChildrenVectors);
+ }
+
+ if (rowPosition.getIsNull() != null) {
+ setFieldNullFlag(rowPosition.getIsNull(), heapRowVector);
+ }
+
+ // Hudi-specific: collapse a present row whose every child is null into a null row, so that a
+ // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the
+ // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely
+ // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the
+ // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes.
+ // positionsCount comes from the Dremel definition/repetition level stream
+ // (NestedPositionUtil#calculateRowOffsets). On a full, non-final batch that stream carries a
+ // one-record lookahead (NestedPrimitiveColumnReader#readAndNewVector reads one value past the
+ // batch in its do/while, and #getLevelDelegation keeps that trailing level for the next batch),
+ // so positionsCount can be one larger than the materialized vector lengths. When inside==true
+ // the row vector is renewed to positionsCount but its children are sized to their value count;
+ // when inside==false the row vector keeps its batch capacity. Either way, iterating all the way
+ // to positionsCount can read one element past a shorter vector and throw
+ // ArrayIndexOutOfBoundsException. Clamp to the shortest vector this loop indexes -- the phantom
+ // trailing position is never surfaced downstream (ParquetColumnarRowSplitReader caps the batch
+ // at num).
+ int rowCount = Math.min(rowPosition.getPositionsCount(), heapRowVector.getLen());
+ for (WritableColumnVector child : finalChildrenVectors) {
+ rowCount = Math.min(rowCount, vectorLength(child));
+ }
+ for (int j = 0; j < rowCount; j++) {
+ if (heapRowVector.isNullAt(j)) {
+ continue;
+ }
+ boolean allChildrenNull = true;
+ for (WritableColumnVector child : finalChildrenVectors) {
+ if (!child.isNullAt(j)) {
+ allChildrenNull = false;
+ break;
+ }
+ }
+ if (allChildrenNull) {
+ heapRowVector.setNullAt(j);
+ }
+ }
+ return Tuple2.of(levelDelegation, heapRowVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readMap(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapMapColumnVector mapVector = (HeapMapColumnVector) vector;
+ mapVector.reset();
+ List<ParquetField> children = field.getChildren();
+ Preconditions.checkArgument(
+ children.size() == 2,
+ "Maps must have two type parameters, found %s",
+ children.size());
+ Tuple2<LevelDelegation, WritableColumnVector> keyTuple =
+ readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true);
+ Tuple2<LevelDelegation, WritableColumnVector> valueTuple =
+ readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true);
+
+ LevelDelegation levelDelegation = keyTuple.f0;
+
+ CollectionPosition collectionPosition =
+ NestedPositionUtil.calculateCollectionOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If map was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1);
+ } else {
+ mapVector.setKeys(keyTuple.f1);
+ mapVector.setValues(valueTuple.f1);
+ }
+
+ if (collectionPosition.getIsNull() != null) {
+ setFieldNullFlag(collectionPosition.getIsNull(), mapVector);
+ }
+
+ mapVector.setLengths(collectionPosition.getLength());
+ mapVector.setOffsets(collectionPosition.getOffsets());
+
+ return Tuple2.of(levelDelegation, mapVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readArray(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapArrayVector arrayVector = (HeapArrayVector) vector;
+ arrayVector.reset();
+ List<ParquetField> children = field.getChildren();
+ Preconditions.checkArgument(
+ children.size() == 1,
+ "Arrays must have a single type parameter, found %s",
+ children.size());
+ Tuple2<LevelDelegation, WritableColumnVector> tuple =
+ readData(children.get(0), readNumber, arrayVector.getChild(), true);
+
+ LevelDelegation levelDelegation = tuple.f0;
+ CollectionPosition collectionPosition =
+ NestedPositionUtil.calculateCollectionOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If array was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1);
+ } else {
+ arrayVector.setChild(tuple.f1);
+ }
+
+ if (collectionPosition.getIsNull() != null) {
+ setFieldNullFlag(collectionPosition.getIsNull(), arrayVector);
+ }
+ arrayVector.setLengths(collectionPosition.getLength());
+ arrayVector.setOffsets(collectionPosition.getOffsets());
+ return Tuple2.of(levelDelegation, arrayVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readPrimitive(
+ ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException {
+ ColumnDescriptor descriptor = field.getDescriptor();
+ NestedPrimitiveColumnReader reader = columnReaders.get(descriptor);
+ if (reader == null) {
+ reader =
+ new NestedPrimitiveColumnReader(
+ descriptor,
+ pages.getPageReader(descriptor),
+ isUtcTimestamp,
+ descriptor.getPrimitiveType(),
+ field.getType());
+ columnReaders.put(descriptor, reader);
+ }
+ WritableColumnVector writableColumnVector =
+ reader.readAndNewVector(readNumber, (WritableColumnVector) vector);
+ return Tuple2.of(reader.getLevelDelegation(), writableColumnVector);
+ }
+
+ /**
+ * The length of the {@code isNull}-backed storage that {@code vector} (a row child) is indexed
+ * against by the null-collapse loop in {@link #readRow}. Every row child is an {@link
+ * AbstractHeapVector} (nested rows/arrays/maps and all non-decimal primitives) or a {@link
+ * ParquetDecimalVector} wrapping one (DECIMAL leaves; see {@code
+ * NestedPrimitiveColumnReader#fillColumnVector}); unwrapping the latter yields an {@code
+ * AbstractHeapVector} in all cases.
+ */
+ private static int vectorLength(ColumnVector vector) {
+ ColumnVector storage =
+ vector instanceof ParquetDecimalVector
+ ? ((ParquetDecimalVector) vector).getVector()
+ : vector;
+ return ((AbstractHeapVector) storage).getLen();
+ }
+
+ private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) {
+ for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) {
+ if (nullFlags[index]) {
+ vector.setNullAt(index);
+ }
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
new file mode 100644
index 000000000000..f6fb14a8d583
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.hudi.table.format.cow.utils.IntArrayList;
+import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
+import org.apache.hudi.table.format.cow.vector.position.LevelDelegation;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.DataPageV1;
+import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.VALUES;
+
+/**
+ * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure.
+ *
+ * <p>Vendored from Apache Flink 2.1 (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package
+ * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList}
+ * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code
+ * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf-
+ * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it
+ * a faithful copy of upstream.
+ */
+@Slf4j
+public class NestedPrimitiveColumnReader implements ColumnReader<WritableColumnVector> {
+ private final IntArrayList repetitionLevelList = new IntArrayList(0);
+ private final IntArrayList definitionLevelList = new IntArrayList(0);
+
+ private final PageReader pageReader;
+ private final ColumnDescriptor descriptor;
+ private final Type type;
+ private final LogicalType logicalType;
+
+ /** The dictionary, if this column has dictionary encoding. */
+ private final ParquetDataColumnReader dictionary;
+
+ /** Maximum definition level for this column. */
+ private final int maxDefLevel;
+
+ private boolean isUtcTimestamp;
+
+ /** Total number of values read. */
+ private long valuesRead;
+
+ /**
+ * value that indicates the end of the current page. That is, if valuesRead ==
+ * endOfPageValueCount, we are at the end of the page.
+ */
+ private long endOfPageValueCount;
+
+ /** If true, the current page is dictionary encoded. */
+ private boolean isCurrentPageDictionaryEncoded;
+
+ private int definitionLevel;
+ private int repetitionLevel;
+
+ /** Repetition/Definition/Value readers. */
+ private IntIterator repetitionLevelColumn;
+
+ private IntIterator definitionLevelColumn;
+ private ParquetDataColumnReader dataColumn;
+
+ /** Total values in the current page. */
+ private int pageValueCount;
+
+ // flag to indicate if there is no data in parquet data page
+ private boolean eof = false;
+
+ private boolean isFirstRow = true;
+
+ private Object lastValue;
+
+ public NestedPrimitiveColumnReader(
+ ColumnDescriptor descriptor,
+ PageReader pageReader,
+ boolean isUtcTimestamp,
+ Type parquetType,
+ LogicalType logicalType)
+ throws IOException {
+ this.descriptor = descriptor;
+ this.type = parquetType;
+ this.pageReader = pageReader;
+ this.maxDefLevel = descriptor.getMaxDefinitionLevel();
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.logicalType = logicalType;
+
+ DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
+ if (dictionaryPage != null) {
+ try {
+ this.dictionary =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ parquetType.asPrimitiveType(),
+ dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } catch (IOException e) {
+ throw new IOException(
+ String.format("Could not decode the dictionary for %s", descriptor), e);
+ }
+ } else {
+ this.dictionary = null;
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+ }
+
+ // Not invoked directly; callers use readAndNewVector instead.
+ @Override
+ public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
+ throw new UnsupportedOperationException("This function should not be called.");
+ }
+
+ public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector)
+ throws IOException {
+ if (isFirstRow) {
+ if (!readValue()) {
+ return vector;
+ }
+ isFirstRow = false;
+ }
+
+ // index to set value.
+ int index = 0;
+ int valueIndex = 0;
+ List<Object> valueList = new ArrayList<>();
+
+ // repeated type need two loops to read data.
+ while (!eof && index < readNumber) {
+ do {
+ valueList.add(lastValue);
+ valueIndex++;
+ } while (readValue() && (repetitionLevel != 0));
+ index++;
+ }
+
+ return fillColumnVector(valueIndex, valueList);
+ }
+
+ public LevelDelegation getLevelDelegation() {
+ int[] repetition = repetitionLevelList.toArray();
+ int[] definition = definitionLevelList.toArray();
+ repetitionLevelList.clear();
+ definitionLevelList.clear();
+ repetitionLevelList.add(repetitionLevel);
+ definitionLevelList.add(definitionLevel);
+ return new LevelDelegation(repetition, definition);
+ }
+
+ private boolean readValue() throws IOException {
+ int left = readPageIfNeed();
+ if (left > 0) {
+ // get the values of repetition and definitionLevel
+ readAndSaveRepetitionAndDefinitionLevels();
+ // read the data if it isn't null
+ if (definitionLevel == maxDefLevel) {
+ if (isCurrentPageDictionaryEncoded) {
+ int dictionaryId = dataColumn.readValueDictionaryId();
+ lastValue = dictionaryDecodeValue(logicalType, dictionaryId);
+ } else {
+ lastValue = readPrimitiveTypedRow(logicalType);
+ }
+ } else {
+ lastValue = null;
+ }
+ return true;
+ } else {
+ eof = true;
+ return false;
+ }
+ }
+
+ private void readAndSaveRepetitionAndDefinitionLevels() {
+ // get the values of repetition and definitionLevel
+ repetitionLevel = repetitionLevelColumn.nextInt();
+ definitionLevel = definitionLevelColumn.nextInt();
+ valuesRead++;
+ repetitionLevelList.add(repetitionLevel);
+ definitionLevelList.add(definitionLevel);
+ }
+
+ private int readPageIfNeed() throws IOException {
+ // Compute the number of values we want to read in this page.
+ int leftInPage = (int) (endOfPageValueCount - valuesRead);
+ if (leftInPage == 0) {
+ // no data left in current page, load data from new page
+ readPage();
+ leftInPage = (int) (endOfPageValueCount - valuesRead);
+ }
+ return leftInPage;
+ }
+
+ private Object readPrimitiveTypedRow(LogicalType category) {
+ switch (category.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ return dataColumn.readBytes();
+ case BOOLEAN:
+ return dataColumn.readBoolean();
+ case TIME_WITHOUT_TIME_ZONE:
+ case DATE:
+ case INTEGER:
+ return dataColumn.readInteger();
+ case TINYINT:
+ return dataColumn.readTinyInt();
+ case SMALLINT:
+ return dataColumn.readSmallInt();
+ case BIGINT:
+ return dataColumn.readLong();
+ case FLOAT:
+ return dataColumn.readFloat();
+ case DOUBLE:
+ return dataColumn.readDouble();
+ case DECIMAL:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT32:
+ return dataColumn.readInteger();
+ case INT64:
+ return dataColumn.readLong();
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ return dataColumn.readBytes();
+ default:
+ throw new RuntimeException(
+ "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType());
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return dataColumn.readTimestamp();
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) {
+ if (dictionaryValue == null) {
+ return null;
+ }
+
+ switch (category.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ return dictionary.readBytes(dictionaryValue);
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ case INTEGER:
+ return dictionary.readInteger(dictionaryValue);
+ case BOOLEAN:
+ return dictionary.readBoolean(dictionaryValue) ? 1 : 0;
+ case DOUBLE:
+ return dictionary.readDouble(dictionaryValue);
+ case FLOAT:
+ return dictionary.readFloat(dictionaryValue);
+ case TINYINT:
+ return dictionary.readTinyInt(dictionaryValue);
+ case SMALLINT:
+ return dictionary.readSmallInt(dictionaryValue);
+ case BIGINT:
+ return dictionary.readLong(dictionaryValue);
+ case DECIMAL:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT32:
+ return dictionary.readInteger(dictionaryValue);
+ case INT64:
+ return dictionary.readLong(dictionaryValue);
+ case FIXED_LEN_BYTE_ARRAY:
+ case BINARY:
+ return dictionary.readBytes(dictionaryValue);
+ default:
+ throw new RuntimeException(
+ "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType());
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return dictionary.readTimestamp(dictionaryValue);
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private WritableColumnVector fillColumnVector(int total, List valueList) {
+ switch (logicalType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ HeapBytesVector heapBytesVector = new HeapBytesVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ byte[] src = ((List<byte[]>) valueList).get(i);
+ if (src == null) {
+ heapBytesVector.setNullAt(i);
+ } else {
+ heapBytesVector.appendBytes(i, src, 0, src.length);
+ }
+ }
+ return heapBytesVector;
+ case BOOLEAN:
+ HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapBooleanVector.setNullAt(i);
+ } else {
+ heapBooleanVector.vector[i] = ((List<Boolean>) valueList).get(i);
+ }
+ }
+ return heapBooleanVector;
+ case TINYINT:
+ HeapByteVector heapByteVector = new HeapByteVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapByteVector.setNullAt(i);
+ } else {
+ heapByteVector.vector[i] = (byte) ((List<Integer>) valueList).get(i).intValue();
+ }
+ }
+ return heapByteVector;
+ case SMALLINT:
+ HeapShortVector heapShortVector = new HeapShortVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapShortVector.setNullAt(i);
+ } else {
+ heapShortVector.vector[i] = (short) ((List<Integer>) valueList).get(i).intValue();
+ }
+ }
+ return heapShortVector;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ HeapIntVector heapIntVector = new HeapIntVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapIntVector.setNullAt(i);
+ } else {
+ heapIntVector.vector[i] = ((List<Integer>) valueList).get(i);
+ }
+ }
+ return heapIntVector;
+ case FLOAT:
+ HeapFloatVector heapFloatVector = new HeapFloatVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapFloatVector.setNullAt(i);
+ } else {
+ heapFloatVector.vector[i] = ((List<Float>) valueList).get(i);
+ }
+ }
+ return heapFloatVector;
+ case BIGINT:
+ HeapLongVector heapLongVector = new HeapLongVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapLongVector.setNullAt(i);
+ } else {
+ heapLongVector.vector[i] = ((List<Long>) valueList).get(i);
+ }
+ }
+ return heapLongVector;
+ case DOUBLE:
+ HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapDoubleVector.setNullAt(i);
+ } else {
+ heapDoubleVector.vector[i] = ((List<Double>) valueList).get(i);
+ }
+ }
+ return heapDoubleVector;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapTimestampVector.setNullAt(i);
+ } else {
+ heapTimestampVector.setTimestamp(i, ((List<TimestampData>) valueList).get(i));
+ }
+ }
+ return heapTimestampVector;
+ case DECIMAL:
+ PrimitiveType.PrimitiveTypeName primitiveTypeName =
+ descriptor.getPrimitiveType().getPrimitiveTypeName();
+ switch (primitiveTypeName) {
+ case INT32:
+ HeapIntVector phiv = new HeapIntVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ phiv.setNullAt(i);
+ } else {
+ phiv.vector[i] = ((List<Integer>) valueList).get(i);
+ }
+ }
+ return new ParquetDecimalVector(phiv);
+ case INT64:
+ HeapLongVector phlv = new HeapLongVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ phlv.setNullAt(i);
+ } else {
+ phlv.vector[i] = ((List<Long>) valueList).get(i);
+ }
+ }
+ return new ParquetDecimalVector(phlv);
+ default:
+ HeapBytesVector phbv = getHeapBytesVector(total, valueList);
+ return new ParquetDecimalVector(phbv);
+ }
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static HeapBytesVector getHeapBytesVector(int total, List valueList) {
+ HeapBytesVector phbv = new HeapBytesVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ byte[] src = ((List<byte[]>) valueList).get(i);
+ if (valueList.get(i) == null) {
+ phbv.setNullAt(i);
+ } else {
+ phbv.appendBytes(i, src, 0, src.length);
+ }
+ }
+ return phbv;
+ }
+
+ protected void readPage() {
+ DataPage page = pageReader.readPage();
+
+ if (page == null) {
+ return;
+ }
+
+ page.accept(
+ new DataPage.Visitor<Void>() {
+ @Override
+ public Void visit(DataPageV1 dataPageV1) {
+ readPageV1(dataPageV1);
+ return null;
+ }
+
+ @Override
+ public Void visit(DataPageV2 dataPageV2) {
+ readPageV2(dataPageV2);
+ return null;
+ }
+ });
+ }
+
+ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount)
+ throws IOException {
+ this.pageValueCount = valueCount;
+ this.endOfPageValueCount = valuesRead + pageValueCount;
+ if (dataEncoding.usesDictionary()) {
+ this.dataColumn = null;
+ if (dictionary == null) {
+ throw new IOException(
+ String.format(
+ "Could not read page in col %s because the dictionary was missing for encoding %s.",
+ descriptor, dataEncoding));
+ }
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getDictionaryBasedValuesReader(
+ descriptor, VALUES, dictionary.getDictionary()),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } else {
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getValuesReader(descriptor, VALUES),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+
+ try {
+ dataColumn.initFromPage(pageValueCount, in);
+ } catch (IOException e) {
+ throw new IOException(String.format("Could not read page in col %s.", descriptor), e);
+ }
+ }
+
+ private void readPageV1(DataPageV1 page) {
+ ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
+ ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL);
+ this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
+ this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
+ try {
+ BytesInput bytes = page.getBytes();
+ log.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount);
+ ByteBufferInputStream in = bytes.toInputStream();
+ log.debug("Reading repetition levels at {}.", in.position());
+ rlReader.initFromPage(pageValueCount, in);
+ log.debug("Reading definition levels at {}.", in.position());
+ dlReader.initFromPage(pageValueCount, in);
+ log.debug("Reading data at {}.", in.position());
+ initDataReader(page.getValueEncoding(), in, page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read page %s in col %s.", page, descriptor), e);
+ }
+ }
+
+ private void readPageV2(DataPageV2 page) {
+ this.pageValueCount = page.getValueCount();
+ this.repetitionLevelColumn =
+ newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels());
+ this.definitionLevelColumn =
+ newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels());
+ try {
+ log.debug(
+ "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount);
+ initDataReader(
+ page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read page %s in col %s.", page, descriptor), e);
+ }
+ }
+
+ private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
+ try {
+ if (maxLevel == 0) {
+ return new NullIntIterator();
+ }
+ return new RLEIntIterator(
+ new RunLengthBitPackingHybridDecoder(
+ BytesUtils.getWidthFromMaxInt(maxLevel),
+ new ByteArrayInputStream(bytes.toByteArray())));
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read levels in page for col %s.", descriptor), e);
+ }
+ }
+
+ /** Utility interface to abstract over different way to read ints with different encodings. */
+ interface IntIterator {
+ int nextInt();
+ }
+
+ /** Reading int from {@link ValuesReader}. */
+ protected static final class ValuesReaderIntIterator implements IntIterator {
+ ValuesReader delegate;
+
+ public ValuesReaderIntIterator(ValuesReader delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int nextInt() {
+ return delegate.readInteger();
+ }
+ }
+
+ /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */
+ protected static final class RLEIntIterator implements IntIterator {
+ RunLengthBitPackingHybridDecoder delegate;
+
+ public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int nextInt() {
+ try {
+ return delegate.readInt();
+ } catch (IOException e) {
+ throw new ParquetDecodingException(e);
+ }
+ }
+ }
+
+ /** Reading zero always. */
+ protected static final class NullIntIterator implements IntIterator {
+ @Override
+ public int nextInt() {
+ return 0;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
new file mode 100644
index 000000000000..d077630b41cf
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil;
+import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.columnar.ColumnarRowData;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReadStore;
+import org.apache.parquet.filter.UnboundRecordFilter;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.io.ColumnIOFactory;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
+import static org.apache.parquet.filter2.compat.FilterCompat.get;
+import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
+
+/**
+ * This reader is used to read a {@link VectorizedColumnBatch} from input split.
+ *
+ * <p>Note: Reference Flink release 1.11.2
+ * {@code org.apache.flink.formats.parquet.vector.ParquetColumnarRowSplitReader}
+ * because it is package scope.
+ */
+public class ParquetColumnarRowSplitReader implements Closeable {
+
+ private final boolean utcTimestamp;
+
+ private final MessageType fileSchema;
+
+ private final LogicalType[] requestedTypes;
+
+ private final MessageType requestedSchema;
+
+ /**
+ * {@link ParquetField} tree per top-level requested column, used by
+ * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List,
+ * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for
+ * nested types. Entries are {@code null} for primitive top-level fields. Built once per split.
+ */
+ private final List<ParquetField> requestedFields;
+
+ /**
+ * The total number of rows this RecordReader will eventually read. The sum of the rows of all
+ * the row groups.
+ */
+ private final long totalRowCount;
+
+ private final WritableColumnVector[] writableVectors;
+
+ private final VectorizedColumnBatch columnarBatch;
+
+ private final ColumnarRowData row;
+
+ private final int batchSize;
+
+ private ParquetFileReader reader;
+
+ /**
+ * For each request column, the reader to read this column. This is NULL if this column is
+ * missing from the file, in which case we populate the attribute with NULL.
+ */
+ private ColumnReader[] columnReaders;
+
+ /**
+ * The number of rows that have been returned.
+ */
+ private long rowsReturned;
+
+ /**
+ * The number of rows that have been reading, including the current in flight row group.
+ */
+ private long totalCountLoadedSoFar;
+
+ // the index of the next row to return
+ private int nextRow;
+
+ // the number of rows in the current batch
+ private int rowsInBatch;
+
+ public ParquetColumnarRowSplitReader(
+ boolean utcTimestamp,
+ boolean caseSensitive,
+ Configuration conf,
+ LogicalType[] selectedTypes,
+ String[] selectedFieldNames,
+ ColumnBatchGenerator generator,
+ int batchSize,
+ Path path,
+ long splitStart,
+ long splitLength,
+ FilterPredicate filterPredicate,
+ UnboundRecordFilter recordFilter) throws IOException {
+ this.utcTimestamp = utcTimestamp;
+ this.batchSize = batchSize;
+ // then we need to apply the predicate push down filter
+ ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength));
+ MessageType fileSchema = footer.getFileMetaData().getSchema();
+ FilterCompat.Filter filter = get(filterPredicate, recordFilter);
+ List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema);
+
+ this.fileSchema = footer.getFileMetaData().getSchema();
+
+ Type[] types = clipParquetSchema(fileSchema, selectedFieldNames, caseSensitive);
+ int[] requestedIndices = IntStream.range(0, types.length).filter(i -> types[i] != null).toArray();
+ Type[] readTypes = Arrays.stream(requestedIndices).mapToObj(i -> types[i]).toArray(Type[]::new);
+
+ this.requestedTypes = Arrays.stream(requestedIndices).mapToObj(i -> selectedTypes[i]).toArray(LogicalType[]::new);
+ this.requestedSchema = Types.buildMessage().addFields(readTypes).named("flink-parquet");
+ this.reader = new ParquetFileReader(
+ conf, footer.getFileMetaData(), path, blocks, requestedSchema.getColumns());
+
+ long totalRowCount = 0;
+ for (BlockMetaData block : blocks) {
+ totalRowCount += block.getRowCount();
+ }
+ this.totalRowCount = totalRowCount;
+ this.nextRow = 0;
+ this.rowsInBatch = 0;
+ this.rowsReturned = 0;
+
+ checkSchema();
+
+ // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across
+ // row groups). Only columns with nested logical type get a non-null entry — primitive columns
+ // still use Hudi's specialized ColumnReaders.
+ MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema);
+ List<RowType.RowField> requestedRowFields = new ArrayList<>(requestedTypes.length);
+ List<String> requestedFieldNames = new ArrayList<>(requestedTypes.length);
+ for (int i = 0; i < requestedTypes.length; i++) {
+ String name = requestedSchema.getFieldName(i);
+ requestedRowFields.add(new RowType.RowField(name, requestedTypes[i]));
+ requestedFieldNames.add(name);
+ }
+ this.requestedFields = ParquetSplitReaderUtil.buildFieldsList(
+ requestedRowFields, requestedFieldNames, messageColumnIO);
+
+ this.writableVectors = createWritableVectors();
+ ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices);
+ this.columnarBatch = generator.generate(columnVectors);
+ this.row = new ColumnarRowData(columnarBatch);
+ }
+
+ /**
+ * Patches the given vectors with nulls.
+ * The vector position that is not requested (or read from file) is patched as null.
+ *
+ * @param fields The total selected fields number
+ * @param vectors The readable vectors
+ * @param indices The requested indices from the selected fields
+ */
+ private static ColumnVector[] patchedVector(int fields, ColumnVector[] vectors, int[] indices) {
+ ColumnVector[] patched = new ColumnVector[fields];
+ for (int i = 0; i < indices.length; i++) {
+ patched[indices[i]] = vectors[i];
+ }
+ return patched;
+ }
+
+ /**
+ * Clips `parquetSchema` according to `fieldNames`.
+ */
+ private static Type[] clipParquetSchema(
+ GroupType parquetSchema, String[] fieldNames, boolean caseSensitive) {
+ Type[] types = new Type[fieldNames.length];
+ if (caseSensitive) {
+ for (int i = 0; i < fieldNames.length; ++i) {
+ String fieldName = fieldNames[i];
+ types[i] = parquetSchema.containsField(fieldName) ? parquetSchema.getType(fieldName) : null;
+ }
+ } else {
+ Map<String, Type> caseInsensitiveFieldMap = new HashMap<>();
+ for (Type type : parquetSchema.getFields()) {
+ caseInsensitiveFieldMap.compute(type.getName().toLowerCase(Locale.ROOT),
+ (key, previousType) -> {
+ if (previousType != null) {
+ throw new FlinkRuntimeException(
+ "Parquet with case insensitive mode should have no duplicate key: " + key);
+ }
+ return type;
+ });
+ }
+ for (int i = 0; i < fieldNames.length; ++i) {
+ Type type = caseInsensitiveFieldMap.get(fieldNames[i].toLowerCase(Locale.ROOT));
+ // TODO clip for array,map,row types.
+ types[i] = type;
+ }
+ }
+
+ return types;
+ }
+
+ private WritableColumnVector[] createWritableVectors() {
+ WritableColumnVector[] columns = new WritableColumnVector[requestedTypes.length];
+ List<Type> types = requestedSchema.getFields();
+ List<ColumnDescriptor> descriptors = requestedSchema.getColumns();
+ for (int i = 0; i < requestedTypes.length; i++) {
+ try {
+ columns[i] = createWritableColumnVector(
+ batchSize,
+ requestedTypes[i],
+ types.get(i),
+ descriptors);
+ } catch (IllegalArgumentException e) {
+ String fieldName = requestedSchema.getFieldName(i);
+ String message = e.getMessage() + " Field name: " + fieldName;
+ throw new IllegalArgumentException(message);
+ }
+ }
+ return columns;
+ }
+
+ /**
+ * Create readable vectors from writable vectors.
+ * Especially for decimal, see {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector}.
+ */
+ private ColumnVector[] createReadableVectors() {
+ ColumnVector[] vectors = new ColumnVector[writableVectors.length];
+ for (int i = 0; i < writableVectors.length; i++) {
+ vectors[i] = requestedTypes[i].getTypeRoot() == LogicalTypeRoot.DECIMAL
+ ? new ParquetDecimalVector(writableVectors[i])
+ : writableVectors[i];
+ }
+ return vectors;
+ }
+
+ private void checkSchema() throws IOException, UnsupportedOperationException {
+ /*
+ * Check that the requested schema is supported.
+ */
+ for (int i = 0; i < requestedSchema.getFieldCount(); ++i) {
+ String[] colPath = requestedSchema.getPaths().get(i);
+ if (fileSchema.containsPath(colPath)) {
+ ColumnDescriptor fd = fileSchema.getColumnDescription(colPath);
+ if (!fd.equals(requestedSchema.getColumns().get(i))) {
+ throw new UnsupportedOperationException("Schema evolution not supported.");
+ }
+ } else {
+ if (requestedSchema.getColumns().get(i).getMaxDefinitionLevel() == 0) {
+ // Column is missing in data but the required data is non-nullable. This file is invalid.
+ throw new IOException("Required column is missing in data file. Col: " + Arrays.toString(colPath));
+ }
+ }
+ }
+ }
+
+ /**
+ * Method used to check if the end of the input is reached.
+ *
+ * @return True if the end is reached, otherwise false.
+ * @throws IOException Thrown, if an I/O error occurred.
+ */
+ public boolean reachedEnd() throws IOException {
+ return !ensureBatch();
+ }
+
+ public RowData nextRecord() {
+ // return the next row
+ row.setRowId(this.nextRow++);
+ return row;
+ }
+
+ /**
+ * Checks if there is at least one row left in the batch to return. If no more row are
+ * available, it reads another batch of rows.
+ *
+ * @return Returns true if there is one more row to return, false otherwise.
+ * @throws IOException throw if an exception happens while reading a batch.
+ */
+ private boolean ensureBatch() throws IOException {
+ if (nextRow >= rowsInBatch) {
+ // Try to read the next batch if rows from the file.
+ if (nextBatch()) {
+ // No more rows available in the Rows array.
+ nextRow = 0;
+ return true;
+ }
+ return false;
+ }
+ // there is at least one Row left in the Rows array.
+ return true;
+ }
+
+ /**
+ * Advances to the next batch of rows. Returns false if there are no more.
+ */
+ private boolean nextBatch() throws IOException {
+ for (WritableColumnVector v : writableVectors) {
+ v.reset();
+ }
+ columnarBatch.setNumRows(0);
+ if (rowsReturned >= totalRowCount) {
+ return false;
+ }
+ if (rowsReturned == totalCountLoadedSoFar) {
+ readNextRowGroup();
+ }
+
+ int num = (int) Math.min(batchSize, totalCountLoadedSoFar - rowsReturned);
+ for (int i = 0; i < columnReaders.length; ++i) {
+ //noinspection unchecked
+ columnReaders[i].readToVector(num, writableVectors[i]);
+ }
+ rowsReturned += num;
+ columnarBatch.setNumRows(num);
+ rowsInBatch = num;
+ return true;
+ }
+
+ private void readNextRowGroup() throws IOException {
+ PageReadStore pages = reader.readNextRowGroup();
+ if (pages == null) {
+ throw new IOException("expecting more rows but reached last block. Read "
+ + rowsReturned + " out of " + totalRowCount);
+ }
+ List<Type> types = requestedSchema.getFields();
+ List<ColumnDescriptor> columns = requestedSchema.getColumns();
+ columnReaders = new ColumnReader[types.size()];
+ for (int i = 0; i < types.size(); ++i) {
+ columnReaders[i] = ParquetSplitReaderUtil.createColumnReader(
+ utcTimestamp,
+ requestedTypes[i],
+ types.get(i),
+ columns,
+ pages,
+ requestedFields.get(i));
+ }
+ totalCountLoadedSoFar += pages.getRowCount();
+ }
+
+ /**
+ * Seek to a particular row number.
+ */
+ public void seekToRow(long rowCount) throws IOException {
+ if (totalCountLoadedSoFar != 0) {
+ throw new UnsupportedOperationException("Only support seek at first.");
+ }
+
+ List<BlockMetaData> blockMetaData = reader.getRowGroups();
+
+ for (BlockMetaData metaData : blockMetaData) {
+ if (metaData.getRowCount() > rowCount) {
+ break;
+ } else {
+ reader.skipNextRowGroup();
+ rowsReturned += metaData.getRowCount();
+ totalCountLoadedSoFar += metaData.getRowCount();
+ rowsInBatch = (int) metaData.getRowCount();
+ nextRow = (int) metaData.getRowCount();
+ rowCount -= metaData.getRowCount();
+ }
+ }
+ for (int i = 0; i < rowCount; i++) {
+ boolean end = reachedEnd();
+ if (end) {
+ throw new RuntimeException("Seek to many rows.");
+ }
+ nextRecord();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (reader != null) {
+ reader.close();
+ reader = null;
+ }
+ }
+
+ /**
+ * Interface to gen {@link VectorizedColumnBatch}.
+ */
+ public interface ColumnBatchGenerator {
+ VectorizedColumnBatch generate(ColumnVector[] readVectors);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
new file mode 100644
index 000000000000..3ff0e0478323
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.TimestampData;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.Dictionary;
+
+import java.io.IOException;
+
+/**
+ * The interface to wrap the underlying Parquet dictionary and non dictionary encoded page reader.
+ */
+public interface ParquetDataColumnReader {
+
+ /**
+ * Initialize the reader by page data.
+ *
+ * @param valueCount value count
+ * @param in page data
+ * @throws IOException
+ */
+ void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException;
+
+ /**
+ * @return the next Dictionary ID from the page
+ */
+ int readValueDictionaryId();
+
+ /**
+ * @return the next Long from the page
+ */
+ long readLong();
+
+ /**
+ * @return the next Integer from the page
+ */
+ int readInteger();
+
+ /**
+ * @return the next SmallInt from the page
+ */
+ int readSmallInt();
+
+ /**
+ * @return the next TinyInt from the page
+ */
+ int readTinyInt();
+
+ /**
+ * @return the next Float from the page
+ */
+ float readFloat();
+
+ /**
+ * @return the next Boolean from the page
+ */
+ boolean readBoolean();
+
+ /**
+ * @return the next String from the page
+ */
+ byte[] readString();
+
+ /**
+ * @return the next Varchar from the page
+ */
+ byte[] readVarchar();
+
+ /**
+ * @return the next Char from the page
+ */
+ byte[] readChar();
+
+ /**
+ * @return the next Bytes from the page
+ */
+ byte[] readBytes();
+
+ /**
+ * @return the next Decimal from the page
+ */
+ byte[] readDecimal();
+
+ /**
+ * @return the next Double from the page
+ */
+ double readDouble();
+
+ /**
+ * @return the next TimestampData from the page
+ */
+ TimestampData readTimestamp();
+
+ /**
+ * @return is data valid
+ */
+ boolean isValid();
+
+ /**
+ * @return the underlying dictionary if current reader is dictionary encoded
+ */
+ Dictionary getDictionary();
+
+ /**
+ * @param id in dictionary
+ * @return the Bytes from the dictionary by id
+ */
+ byte[] readBytes(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Float from the dictionary by id
+ */
+ float readFloat(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Double from the dictionary by id
+ */
+ double readDouble(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Integer from the dictionary by id
+ */
+ int readInteger(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Long from the dictionary by id
+ */
+ long readLong(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Small Int from the dictionary by id
+ */
+ int readSmallInt(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the tiny int from the dictionary by id
+ */
+ int readTinyInt(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Boolean from the dictionary by id
+ */
+ boolean readBoolean(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Decimal from the dictionary by id
+ */
+ byte[] readDecimal(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the TimestampData from the dictionary by id
+ */
+ TimestampData readTimestamp(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the String from the dictionary by id
+ */
+ byte[] readString(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Varchar from the dictionary by id
+ */
+ byte[] readVarchar(int id);
+
+ /**
+ * @param id in dictionary
+ * @return the Char from the dictionary by id
+ */
+ byte[] readChar(int id);
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
new file mode 100644
index 000000000000..fcdd5d382b98
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.TimestampData;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS;
+import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY;
+import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.NANOS_PER_MILLISECOND;
+import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.NANOS_PER_SECOND;
+
+/**
+ * Parquet file has self-describing schema which may differ from the user required schema (e.g.
+ * schema evolution). This factory is used to retrieve user required typed data via corresponding
+ * reader which reads the underlying data.
+ */
+public final class ParquetDataColumnReaderFactory {
+
+ private ParquetDataColumnReaderFactory() {
+ }
+
+ /**
+ * default reader for {@link ParquetDataColumnReader}.
+ */
+ public static class DefaultParquetDataColumnReader implements ParquetDataColumnReader {
+ protected ValuesReader valuesReader;
+ protected Dictionary dict;
+
+ // After the data is read in the parquet type, isValid will be set to true if the data can
+ // be returned in the type defined in HMS. Otherwise isValid is set to false.
+ boolean isValid = true;
+
+ public DefaultParquetDataColumnReader(ValuesReader valuesReader) {
+ this.valuesReader = valuesReader;
+ }
+
+ public DefaultParquetDataColumnReader(Dictionary dict) {
+ this.dict = dict;
+ }
+
+ public boolean isValid() {
+ return isValid;
+ }
+
+ @Override
+ public void initFromPage(int i, ByteBufferInputStream in) throws IOException {
+ valuesReader.initFromPage(i, in);
+ }
+
+ @Override
+ public boolean readBoolean() {
+ return valuesReader.readBoolean();
+ }
+
+ @Override
+ public boolean readBoolean(int id) {
+ return dict.decodeToBoolean(id);
+ }
+
+ @Override
+ public byte[] readString(int id) {
+ return dict.decodeToBinary(id).getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readString() {
+ return valuesReader.readBytes().getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readVarchar() {
+ // we need to enforce the size here even the types are the same
+ return valuesReader.readBytes().getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readVarchar(int id) {
+ return dict.decodeToBinary(id).getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readChar() {
+ return valuesReader.readBytes().getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readChar(int id) {
+ return dict.decodeToBinary(id).getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readBytes() {
+ return valuesReader.readBytes().getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readBytes(int id) {
+ return dict.decodeToBinary(id).getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readDecimal() {
+ return valuesReader.readBytes().getBytesUnsafe();
+ }
+
+ @Override
+ public byte[] readDecimal(int id) {
+ return dict.decodeToBinary(id).getBytesUnsafe();
+ }
+
+ @Override
+ public float readFloat() {
+ return valuesReader.readFloat();
+ }
+
+ @Override
+ public float readFloat(int id) {
+ return dict.decodeToFloat(id);
+ }
+
+ @Override
+ public double readDouble() {
+ return valuesReader.readDouble();
+ }
+
+ @Override
+ public double readDouble(int id) {
+ return dict.decodeToDouble(id);
+ }
+
+ @Override
+ public TimestampData readTimestamp() {
+ throw new RuntimeException("Unsupported operation");
+ }
+
+ @Override
+ public TimestampData readTimestamp(int id) {
+ throw new RuntimeException("Unsupported operation");
+ }
+
+ @Override
+ public int readInteger() {
+ return valuesReader.readInteger();
+ }
+
+ @Override
+ public int readInteger(int id) {
+ return dict.decodeToInt(id);
+ }
+
+ @Override
+ public long readLong(int id) {
+ return dict.decodeToLong(id);
+ }
+
+ @Override
+ public long readLong() {
+ return valuesReader.readLong();
+ }
+
+ @Override
+ public int readSmallInt() {
+ return valuesReader.readInteger();
+ }
+
+ @Override
+ public int readSmallInt(int id) {
+ return dict.decodeToInt(id);
+ }
+
+ @Override
+ public int readTinyInt() {
+ return valuesReader.readInteger();
+ }
+
+ @Override
+ public int readTinyInt(int id) {
+ return dict.decodeToInt(id);
+ }
+
+ @Override
+ public int readValueDictionaryId() {
+ return valuesReader.readValueDictionaryId();
+ }
+
+ public void skip() {
+ valuesReader.skip();
+ }
+
+ @Override
+ public Dictionary getDictionary() {
+ return dict;
+ }
+ }
+
+ /**
+ * The reader who reads from the underlying Timestamp value value.
+ */
+ public static class TypesFromInt96PageReader extends DefaultParquetDataColumnReader {
+ private final boolean isUtcTimestamp;
+
+ public TypesFromInt96PageReader(ValuesReader realReader, boolean isUtcTimestamp) {
+ super(realReader);
+ this.isUtcTimestamp = isUtcTimestamp;
+ }
+
+ public TypesFromInt96PageReader(Dictionary dict, boolean isUtcTimestamp) {
+ super(dict);
+ this.isUtcTimestamp = isUtcTimestamp;
+ }
+
+ private TimestampData convert(Binary binary) {
+ ByteBuffer buf = binary.toByteBuffer();
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ long timeOfDayNanos = buf.getLong();
+ int julianDay = buf.getInt();
+ return int96ToTimestamp(isUtcTimestamp, timeOfDayNanos, julianDay);
+ }
+
+ @Override
+ public TimestampData readTimestamp(int id) {
+ return convert(dict.decodeToBinary(id));
+ }
+
+ @Override
+ public TimestampData readTimestamp() {
+ return convert(valuesReader.readBytes());
+ }
+ }
+
+ /**
+ * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard
+ * timestamp encoding defined by Parquet's
+ * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy
+ * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations.
+ * (The older INT96 encoding is marked deprecated by the Parquet format spec — see
+ * <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp">
+ * LogicalTypes.md</a> — but is still supported here via {@link TypesFromInt96PageReader} for
+ * backwards compatibility with files written by older Hive / Spark / Impala versions.)
+ *
+ * <p>Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a
+ * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use
+ * {@link Int64TimestampColumnReader} for batched-vector efficiency.
+ */
+ public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader {
+ private final boolean isUtcTimestamp;
+ private final ChronoUnit chronoUnit;
+
+ public TypesFromInt64PageReader(
+ ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) {
+ super(realReader);
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.chronoUnit = chronoUnit;
+ }
+
+ public TypesFromInt64PageReader(
+ Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) {
+ super(dict);
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.chronoUnit = chronoUnit;
+ }
+
+ @Override
+ public TimestampData readTimestamp() {
+ return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit);
+ }
+
+ @Override
+ public TimestampData readTimestamp(int id) {
+ return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit);
+ }
+ }
+
+ private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(
+ boolean isDictionary,
+ PrimitiveType parquetType,
+ Dictionary dictionary,
+ ValuesReader valuesReader,
+ boolean isUtcTimestamp) {
+ PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName();
+ if (typeName == PrimitiveType.PrimitiveTypeName.INT96) {
+ return isDictionary
+ ? new TypesFromInt96PageReader(dictionary, isUtcTimestamp)
+ : new TypesFromInt96PageReader(valuesReader, isUtcTimestamp);
+ }
+ if (typeName == PrimitiveType.PrimitiveTypeName.INT64) {
+ ChronoUnit unit = resolveInt64TimestampUnit(parquetType);
+ if (unit != null) {
+ return isDictionary
+ ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit)
+ : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit);
+ }
+ }
+ return isDictionary
+ ? new DefaultParquetDataColumnReader(dictionary)
+ : new DefaultParquetDataColumnReader(valuesReader);
+ }
+
+ /**
+ * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the
+ * column is a plain INT64 (not a timestamp).
+ *
+ * <p>Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and
+ * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS}
+ * encodings.
+ */
+ private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) {
+ LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation();
+ if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
+ LogicalTypeAnnotation.TimeUnit unit =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit();
+ switch (unit) {
+ case MILLIS:
+ return ChronoUnit.MILLIS;
+ case MICROS:
+ return ChronoUnit.MICROS;
+ case NANOS:
+ return ChronoUnit.NANOS;
+ default:
+ return null;
+ }
+ }
+ OriginalType originalType = parquetType.getOriginalType();
+ if (originalType == OriginalType.TIMESTAMP_MILLIS) {
+ return ChronoUnit.MILLIS;
+ }
+ if (originalType == OriginalType.TIMESTAMP_MICROS) {
+ return ChronoUnit.MICROS;
+ }
+ return null;
+ }
+
+ private static TimestampData int64ToTimestamp(
+ boolean isUtcTimestamp, long value, ChronoUnit unit) {
+ Instant instant = Instant.EPOCH.plus(value, unit);
+ if (isUtcTimestamp) {
+ return TimestampData.fromInstant(instant);
+ }
+ return TimestampData.fromTimestamp(Timestamp.from(instant));
+ }
+
+ public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary(
+ PrimitiveType parquetType, Dictionary realReader, boolean isUtcTimestamp) {
+ return getDataColumnReaderByTypeHelper(true, parquetType, realReader, null, isUtcTimestamp);
+ }
+
+ public static ParquetDataColumnReader getDataColumnReaderByType(
+ PrimitiveType parquetType, ValuesReader realReader, boolean isUtcTimestamp) {
+ return getDataColumnReaderByTypeHelper(
+ false, parquetType, null, realReader, isUtcTimestamp);
+ }
+
+ private static TimestampData int96ToTimestamp(
+ boolean isUtcTimestamp, long nanosOfDay, int julianDay) {
+ long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND);
+
+ if (isUtcTimestamp) {
+ int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND);
+ return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond);
+ } else {
+ Timestamp timestamp = new Timestamp(millisecond);
+ timestamp.setNanos((int) (nanosOfDay % NANOS_PER_SECOND));
+ return TimestampData.fromTimestamp(timestamp);
+ }
+ }
+
+ private static long julianDayToMillis(int julianDay) {
+ return (julianDay - JULIAN_EPOCH_OFFSET_DAYS) * MILLIS_IN_DAY;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
new file mode 100644
index 000000000000..0bf4cebee292
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BytePacker;
+import org.apache.parquet.column.values.bitpacking.Packer;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Run length decoder for data and dictionary ids.
+ * See https://github.com/apache/parquet-format/blob/master/Encodings.md
+ * See {@link RunLengthBitPackingHybridDecoder}.
+ *
+ * <p>Note: Reference Flink release 1.11.2
+ * {@code org.apache.flink.formats.parquet.vector.reader.RunLengthDecoder}
+ * because it is package scope.
+ */
+final class RunLengthDecoder {
+
+ /**
+ * If true, the bit width is fixed. This decoder is used in different places and this also
+ * controls if we need to read the bitwidth from the beginning of the data stream.
+ */
+ private final boolean fixedWidth;
+ private final boolean readLength;
+
+ // Encoded data.
+ private ByteBufferInputStream in;
+
+ // bit/byte width of decoded data and utility to batch unpack them.
+ private int bitWidth;
+ private int bytesWidth;
+ private BytePacker packer;
+
+ // Current decoding mode and values
+ MODE mode;
+ int currentCount;
+ int currentValue;
+
+ // Buffer of decoded values if the values are PACKED.
+ int[] currentBuffer = new int[16];
+ int currentBufferIdx = 0;
+
+ RunLengthDecoder() {
+ this.fixedWidth = false;
+ this.readLength = false;
+ }
+
+ RunLengthDecoder(int bitWidth) {
+ this.fixedWidth = true;
+ this.readLength = bitWidth != 0;
+ initWidthAndPacker(bitWidth);
+ }
+
+ RunLengthDecoder(int bitWidth, boolean readLength) {
+ this.fixedWidth = true;
+ this.readLength = readLength;
+ initWidthAndPacker(bitWidth);
+ }
+
+ /**
+ * Init from input stream.
+ */
+ void initFromStream(int valueCount, ByteBufferInputStream in) throws IOException {
+ this.in = in;
+ if (fixedWidth) {
+ // initialize for repetition and definition levels
+ if (readLength) {
+ int length = readIntLittleEndian();
+ this.in = in.sliceStream(length);
+ }
+ } else {
+ // initialize for values
+ if (in.available() > 0) {
+ initWidthAndPacker(in.read());
+ }
+ }
+ if (bitWidth == 0) {
+ // 0 bit width, treat this as an RLE run of valueCount number of 0's.
+ this.mode = MODE.RLE;
+ this.currentCount = valueCount;
+ this.currentValue = 0;
+ } else {
+ this.currentCount = 0;
+ }
+ }
+
+ /**
+ * Initializes the internal state for decoding ints of `bitWidth`.
+ */
+ private void initWidthAndPacker(int bitWidth) {
+ Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
+ this.bitWidth = bitWidth;
+ this.bytesWidth = BytesUtils.paddedByteCountFromBits(bitWidth);
+ this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ }
+
+ int readInteger() {
+ if (this.currentCount == 0) {
+ this.readNextGroup();
+ }
+
+ this.currentCount--;
+ switch (mode) {
+ case RLE:
+ return this.currentValue;
+ case PACKED:
+ return this.currentBuffer[currentBufferIdx++];
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /**
+ * Decoding for dictionary ids. The IDs are populated into `values` and the nullability is
+ * populated into `nulls`.
+ */
+ void readDictionaryIds(
+ int total,
+ WritableIntVector values,
+ WritableColumnVector nulls,
+ int rowId,
+ int level,
+ RunLengthDecoder data) {
+ int left = total;
+ while (left > 0) {
+ if (this.currentCount == 0) {
+ this.readNextGroup();
+ }
+ int n = Math.min(left, this.currentCount);
+ switch (mode) {
+ case RLE:
+ if (currentValue == level) {
+ data.readDictionaryIdData(n, values, rowId);
+ } else {
+ nulls.setNulls(rowId, n);
+ }
+ break;
+ case PACKED:
+ for (int i = 0; i < n; ++i) {
+ if (currentBuffer[currentBufferIdx++] == level) {
+ values.setInt(rowId + i, data.readInteger());
+ } else {
+ nulls.setNullAt(rowId + i);
+ }
+ }
+ break;
+ default:
+ throw new AssertionError();
+ }
+ rowId += n;
+ left -= n;
+ currentCount -= n;
+ }
+ }
+
+ /**
+ * It is used to decode dictionary IDs.
+ */
+ private void readDictionaryIdData(int total, WritableIntVector c, int rowId) {
+ int left = total;
+ while (left > 0) {
+ if (this.currentCount == 0) {
+ this.readNextGroup();
+ }
+ int n = Math.min(left, this.currentCount);
+ switch (mode) {
+ case RLE:
+ c.setInts(rowId, n, currentValue);
+ break;
+ case PACKED:
+ c.setInts(rowId, n, currentBuffer, currentBufferIdx);
+ currentBufferIdx += n;
+ break;
+ default:
+ throw new AssertionError();
+ }
+ rowId += n;
+ left -= n;
+ currentCount -= n;
+ }
+ }
+
+ /**
+ * Reads the next varint encoded int.
+ */
+ private int readUnsignedVarInt() throws IOException {
+ int value = 0;
+ int shift = 0;
+ int b;
+ do {
+ b = in.read();
+ value |= (b & 0x7F) << shift;
+ shift += 7;
+ } while ((b & 0x80) != 0);
+ return value;
+ }
+
+ /**
+ * Reads the next 4 byte little endian int.
+ */
+ private int readIntLittleEndian() throws IOException {
+ int ch4 = in.read();
+ int ch3 = in.read();
+ int ch2 = in.read();
+ int ch1 = in.read();
+ return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + ch4);
+ }
+
+ /**
+ * Reads the next byteWidth little endian int.
+ */
+ private int readIntLittleEndianPaddedOnBitWidth() throws IOException {
+ switch (bytesWidth) {
+ case 0:
+ return 0;
+ case 1:
+ return in.read();
+ case 2: {
+ int ch2 = in.read();
+ int ch1 = in.read();
+ return (ch1 << 8) + ch2;
+ }
+ case 3: {
+ int ch3 = in.read();
+ int ch2 = in.read();
+ int ch1 = in.read();
+ return (ch1 << 16) + (ch2 << 8) + ch3;
+ }
+ case 4: {
+ return readIntLittleEndian();
+ }
+ default:
+ throw new RuntimeException("Unreachable");
+ }
+ }
+
+ /**
+ * Reads the next group.
+ */
+ void readNextGroup() {
+ try {
+ int header = readUnsignedVarInt();
+ this.mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED;
+ switch (mode) {
+ case RLE:
+ this.currentCount = header >>> 1;
+ this.currentValue = readIntLittleEndianPaddedOnBitWidth();
+ return;
+ case PACKED:
+ int numGroups = header >>> 1;
+ this.currentCount = numGroups * 8;
+
+ if (this.currentBuffer.length < this.currentCount) {
+ this.currentBuffer = new int[this.currentCount];
+ }
+ currentBufferIdx = 0;
+ int valueIndex = 0;
+ while (valueIndex < this.currentCount) {
+ // values are bit packed 8 at a time, so reading bitWidth will always work
+ ByteBuffer buffer = in.slice(bitWidth);
+ this.packer.unpack8Values(buffer, buffer.position(), this.currentBuffer, valueIndex);
+ valueIndex += 8;
+ }
+ return;
+ default:
+ throw new ParquetDecodingException("not a valid mode " + this.mode);
+ }
+ } catch (IOException e) {
+ throw new ParquetDecodingException("Failed to read from input stream", e);
+ }
+ }
+
+ enum MODE {
+ RLE,
+ PACKED
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
new file mode 100644
index 000000000000..0f5e00779a2f
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+
+/**
+ * Field that represent parquet's field type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetField}).
+ */
+public abstract class ParquetField {
+ private final LogicalType type;
+ private final int repetitionLevel;
+ private final int definitionLevel;
+ private final boolean required;
+
+ public ParquetField(
+ LogicalType type, int repetitionLevel, int definitionLevel, boolean required) {
+ this.type = type;
+ this.repetitionLevel = repetitionLevel;
+ this.definitionLevel = definitionLevel;
+ this.required = required;
+ }
+
+ public LogicalType getType() {
+ return type;
+ }
+
+ public int getRepetitionLevel() {
+ return repetitionLevel;
+ }
+
+ public int getDefinitionLevel() {
+ return definitionLevel;
+ }
+
+ public boolean isRequired() {
+ return required;
+ }
+
+ @Override
+ public String toString() {
+ return "Field{"
+ + "type="
+ + type
+ + ", repetitionLevel="
+ + repetitionLevel
+ + ", definitionLevel="
+ + definitionLevel
+ + ", required="
+ + required
+ + '}';
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
new file mode 100644
index 000000000000..f91dcca965d6
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Field that represent parquet's Group Field.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension:
+ * entries in the {@code children} list may be {@code null} to denote a Row child that is absent
+ * from the parquet file but present in the requested logical schema (schema evolution). This
+ * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees.
+ */
+public class ParquetGroupField extends ParquetField {
+
+ private final List<ParquetField> children;
+
+ public ParquetGroupField(
+ LogicalType type,
+ int repetitionLevel,
+ int definitionLevel,
+ boolean required,
+ List<ParquetField> children) {
+ super(type, repetitionLevel, definitionLevel, required);
+ // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for
+ // schema-evolution missing children in ROW types.
+ this.children =
+ Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null")));
+ }
+
+ /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */
+ public List<ParquetField> getChildren() {
+ return children;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
new file mode 100644
index 000000000000..f6af6f9ff479
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.parquet.column.ColumnDescriptor;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Field that represent parquet's primitive field.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}).
+ */
+public class ParquetPrimitiveField extends ParquetField {
+
+ private final ColumnDescriptor descriptor;
+ private final int id;
+
+ public ParquetPrimitiveField(
+ LogicalType type, boolean required, ColumnDescriptor descriptor, int id) {
+ super(
+ type,
+ descriptor.getMaxRepetitionLevel(),
+ descriptor.getMaxDefinitionLevel(),
+ required);
+ this.descriptor = requireNonNull(descriptor, "descriptor is required");
+ this.id = id;
+ }
+
+ public ColumnDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ public int getId() {
+ return id;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/CatalogUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/CatalogUtils.java
new file mode 100644
index 000000000000..a24d34df1932
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/CatalogUtils.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.catalog.CatalogTable;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Adapter utils to create {@link CatalogTable}.
+ */
+public class CatalogUtils {
+ public static CatalogTable createCatalogTable(Schema schema, List<String> partitionKeys, Map<String, String> options, @Nullable String comment) {
+ return CatalogTable.newBuilder().schema(schema).comment(comment).partitionKeys(partitionKeys).options(options).build();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/DataTypeUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/DataTypeUtils.java
new file mode 100644
index 000000000000..bf86cc90a384
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/DataTypeUtils.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.legacy.types.logical.TypeInformationRawType;
+import org.apache.flink.table.types.AtomicDataType;
+import org.apache.flink.table.types.DataType;
+
+/**
+ * Adapter utils to provide {@link DataType} utilities.
+ */
+public class DataTypeUtils {
+ public static <T> DataType createAtomicRawType(Boolean isNullable, TypeInformation<T> typeInfo) {
+ return new AtomicDataType(new TypeInformationRawType<>(isNullable, typeInfo));
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/RuntimeContextUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/RuntimeContextUtils.java
new file mode 100644
index 000000000000..f1bd7fdcc8e1
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/RuntimeContextUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.functions.RuntimeContext;
+
+import java.util.Map;
+
+import static org.apache.flink.configuration.PipelineOptions.AUTO_WATERMARK_INTERVAL;
+
+/**
+ * Adapter utils for {@code RuntimeContext} to solve API compatibilities issue.
+ */
+public class RuntimeContextUtils {
+
+ public static JobID getJobId(RuntimeContext runtimeContext) {
+ return runtimeContext.getJobInfo().getJobId();
+ }
+
+ public static int getAttemptNumber(RuntimeContext runtimeContext) {
+ return runtimeContext.getTaskInfo().getAttemptNumber();
+ }
+
+ public static int getIndexOfThisSubtask(RuntimeContext runtimeContext) {
+ return runtimeContext.getTaskInfo().getIndexOfThisSubtask();
+ }
+
+ public static int getMaxNumberOfParallelSubtasks(RuntimeContext runtimeContext) {
+ return runtimeContext.getTaskInfo().getMaxNumberOfParallelSubtasks();
+ }
+
+ public static int getNumberOfParallelSubtasks(RuntimeContext runtimeContext) {
+ return runtimeContext.getTaskInfo().getNumberOfParallelSubtasks();
+ }
+
+ public static long getWatermarkInternal(RuntimeContext runtimeContext) {
+ Map<String, String> jobParameters = runtimeContext.getGlobalJobParameters();
+ return Long.parseLong(jobParameters.getOrDefault(AUTO_WATERMARK_INTERVAL.key(),
+ AUTO_WATERMARK_INTERVAL.defaultValue().toMillis() + ""));
+ }
+
+ public static boolean isObjectReuseEnabled(RuntimeContext runtimeContext) {
+ return runtimeContext.isObjectReuseEnabled();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/SourceContextUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/SourceContextUtils.java
new file mode 100644
index 000000000000..cd306ec22f53
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/SourceContextUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.hudi.adapter.SourceFunctionAdapter;
+
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamSourceContexts;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+
+/**
+ * Adapter utils to get {@code SourceContext}.
+ */
+public class SourceContextUtils {
+
+ public static <O> SourceFunctionAdapter.SourceContext<O> getSourceContext(
+ StreamConfig streamConfig,
+ ProcessingTimeService processingTimeService,
+ Output<StreamRecord<O>> output,
+ long watermarkInterval) {
+ return StreamSourceContexts.getSourceContext(
+ processingTimeService,
+ new Object(), // no actual locking needed
+ output,
+ watermarkInterval,
+ -1,
+ true);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StateTtlConfigUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StateTtlConfigUtils.java
new file mode 100644
index 000000000000..dc92ca727ef9
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StateTtlConfigUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.api.common.state.StateTtlConfig;
+
+import java.time.Duration;
+
+/**
+ * Adapter utils to create {@link StateTtlConfig}.
+ */
+public class StateTtlConfigUtils {
+ public static StateTtlConfig createTtlConfig(long ttl) {
+ return StateTtlConfig.newBuilder(Duration.ofMillis(ttl)).build();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StreamerUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StreamerUtils.java
new file mode 100644
index 000000000000..f9a61bc4dcac
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/utils/StreamerUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.connector.kafka.source.KafkaSource;
+import org.apache.flink.formats.common.TimestampFormat;
+import org.apache.flink.formats.json.JsonRowDataDeserializationSchema;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+
+import java.util.Properties;
+
+/**
+ * Adapter utils to create {@link DataStream} for Kafka source.
+ */
+public class StreamerUtils {
+ public static DataStream<RowData> createKafkaStream(
+ StreamExecutionEnvironment env,
+ RowType rowType,
+ String topic,
+ Properties props) {
+ return env.fromSource(
+ KafkaSource.<RowData>builder()
+ .setBootstrapServers(props.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG))
+ .setValueOnlyDeserializer(new JsonRowDataDeserializationSchema(
+ rowType,
+ InternalTypeInfo.of(rowType),
+ false,
+ true,
+ TimestampFormat.ISO_8601
+ ))
+ .setTopics(topic)
+ .build(),
+ WatermarkStrategy.noWatermarks(),
+ "kafka_source")
+ .uid("uid_kafka_source");
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/NoFetchingInput.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/NoFetchingInput.java
new file mode 100644
index 000000000000..7790dddea46c
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/NoFetchingInput.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import com.esotericsoftware.kryo.KryoException;
+import com.esotericsoftware.kryo.io.Input;
+import org.apache.flink.annotation.Internal;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A customized Kryo {@link Input} used for deserializing in {@code KryoSerializer}.
+ */
+@Internal
+public class NoFetchingInput extends Input {
+ public NoFetchingInput(InputStream inputStream) {
+ super(inputStream, 8);
+ }
+
+ @Override
+ public int read() throws KryoException {
+ require(1);
+ return buffer[position++] & 0xFF;
+ }
+
+ @Override
+ public boolean canReadInt() throws KryoException {
+ throw new UnsupportedOperationException("NoFetchingInput cannot prefetch data.");
+ }
+
+ @Override
+ public boolean canReadLong() throws KryoException {
+ throw new UnsupportedOperationException("NoFetchingInput cannot prefetch data.");
+ }
+
+ /**
+ * Require makes sure that at least required number of bytes are kept in the buffer. If not,
+ * then it will load exactly the difference between required and currently available number of
+ * bytes. Thus, it will only load the data which is required and never prefetch data.
+ *
+ * @param required the number of bytes being available in the buffer
+ * @return The number of bytes remaining in the buffer, which will be at least <code>required
+ * </code> bytes.
+ * @throws KryoException
+ */
+ @Override
+ protected int require(int required) throws KryoException {
+ // The main change between this and Kryo 5 Input.require is this will never read more bytes
+ // than required.
+ // There are also formatting changes to be compliant with the Flink project styling rules.
+ int remaining = limit - position;
+ if (remaining >= required) {
+ return remaining;
+ }
+ if (required > capacity) {
+ throw new KryoException(
+ "Buffer too small: capacity: " + capacity + ", required: " + required);
+ }
+
+ int count;
+ // Try to fill the buffer.
+ if (remaining > 0) {
+ // Logical change 1 (from Kryo Input.require): "capacity - limit" -> "required - limit"
+ count = fill(buffer, limit, required - limit);
+ if (count == -1) {
+ throw new KryoException("Buffer underflow.");
+ }
+ remaining += count;
+ if (remaining >= required) {
+ limit += count;
+ return remaining;
+ }
+ }
+
+ // Was not enough, compact and try again.
+ System.arraycopy(buffer, position, buffer, 0, remaining);
+ total += position;
+ position = 0;
+
+ do {
+ // Logical change 2 (from Kryo Input.require): "capacity - remaining" -> "required -
+ // remaining"
+ count = fill(buffer, remaining, required - remaining);
+ if (count == -1) {
+ throw new KryoException("Buffer underflow.");
+ }
+ remaining += count;
+ } while (remaining < required);
+
+ limit = remaining;
+ return remaining;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int count) throws KryoException {
+ if (bytes == null) {
+ throw new IllegalArgumentException("bytes cannot be null.");
+ }
+
+ try {
+ return inputStream.read(bytes, offset, count);
+ } catch (IOException ex) {
+ throw new KryoException(ex);
+ }
+ }
+
+ @Override
+ public void skip(int count) throws KryoException {
+ try {
+ inputStream.skip(count);
+ } catch (IOException ex) {
+ throw new KryoException(ex);
+ }
+ }
+
+ @Override
+ public void readBytes(byte[] bytes, int offset, int count) throws KryoException {
+ if (bytes == null) {
+ throw new IllegalArgumentException("bytes cannot be null.");
+ }
+
+ if (count == 0) {
+ return;
+ }
+
+ try {
+ int bytesRead = 0;
+ int c;
+
+ while (true) {
+ c = inputStream.read(bytes, offset + bytesRead, count - bytesRead);
+
+ if (c == -1) {
+ throw new KryoException(new EOFException("No more bytes left."));
+ }
+
+ bytesRead += c;
+
+ if (bytesRead == count) {
+ break;
+ }
+ }
+ } catch (IOException ex) {
+ throw new KryoException(ex);
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
new file mode 100644
index 000000000000..5a1f4a4a8517
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.hudi.common.util.SerializationUtils;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.flink.api.common.serialization.SerializerConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.NoFetchingInput;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * This class is introduced to override the Flink {@link KryoSerializer} within same package path for
+ * testing scope only.
+ *
+ * <p>Flink 2.0 uses Kryo 5.6.2 for {@link KryoSerializer}, but tests in hudi repository uses lower
+ * version 4.0.2. To make it compatible regarding Kryo APIs, a simplified {@link KryoSerializer} is
+ * implemented here to overwrite the one in Flink, and it can be removed when version of Kryo in hudi
+ * is upgraded to 5.6.2 or higher.
+ */
+public class KryoSerializer<T> extends TypeSerializer<T> {
+
+ private final Class<T> type;
+ private static final ThreadLocal<Kryo> KRYO_REF =
+ ThreadLocal.withInitial(() -> {
+ SerializationUtils.KryoInstantiator kryoInstantiator = new SerializationUtils.KryoInstantiator();
+ return kryoInstantiator.newKryo();
+ });
+
+ public KryoSerializer(Class<T> type, SerializerConfig serializerConfig) {
+ this.type = type;
+ }
+
+ private Kryo getKryo() {
+ return KRYO_REF.get();
+ }
+
+ @Override
+ public boolean isImmutableType() {
+ return false;
+ }
+
+ @Override
+ public TypeSerializer<T> duplicate() {
+ return new KryoSerializer<>(type, null);
+ }
+
+ @Override
+ public T createInstance() {
+ return getKryo().newInstance(type);
+ }
+
+ @Override
+ public T copy(T from) {
+ if (from == null) {
+ return null;
+ }
+ return getKryo().copy(from);
+ }
+
+ @Override
+ public T copy(T from, T reuse) {
+ return copy(from);
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public void serialize(T record, DataOutputView target) {
+ DataOutputViewStream outputStream = new DataOutputViewStream(target);
+ try (Output output = new Output(outputStream)) {
+ getKryo().writeClassAndObject(output, record);
+ }
+ }
+
+ @Override
+ public T deserialize(DataInputView source) {
+ DataInputViewStream inputStream = new DataInputViewStream(source);
+ try (Input input = new NoFetchingInput(inputStream)) {
+ return (T) getKryo().readClassAndObject(input);
+ }
+ }
+
+ @Override
+ public T deserialize(T reuse, DataInputView source) throws IOException {
+ return deserialize(source);
+ }
+
+ @Override
+ public void copy(DataInputView source, DataOutputView target) throws IOException {
+ T tmp = deserialize(source);
+ serialize(tmp, target);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof KryoSerializer) {
+ KryoSerializer<?> other = (KryoSerializer<?>) obj;
+
+ return type == other.type;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type);
+ }
+
+ @Override
+ public TypeSerializerSnapshot<T> snapshotConfiguration() {
+ return new KryoSerializerSnapshot<>(type);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
new file mode 100644
index 000000000000..aac9a4651aff
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerSnapshot.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime.kryo;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+
+/**
+ * An implementation of {@link TypeSerializerSnapshot} used to snapshot {@link KryoSerializer}.
+ */
+public class KryoSerializerSnapshot<T> implements TypeSerializerSnapshot<T> {
+
+ private Class<T> type;
+
+ public KryoSerializerSnapshot() {
+ }
+
+ public KryoSerializerSnapshot(Class<T> type) {
+ this.type = type;
+ }
+
+ @Override
+ public int getCurrentVersion() {
+ return 0;
+ }
+
+ @Override
+ public void writeSnapshot(DataOutputView out) throws IOException {
+ out.writeUTF(type.getName());
+ }
+
+ @Override
+ public void readSnapshot(int readVersion, DataInputView in, ClassLoader cl) throws IOException {
+ this.type = readTypeClass(in, cl);
+ }
+
+ @Override
+ public TypeSerializer<T> restoreSerializer() {
+ return new KryoSerializer<>(type, null);
+ }
+
+ @Override
+ public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializerSnapshot oldSerializerSnapshot) {
+ return TypeSerializerSchemaCompatibility.compatibleAsIs();
+ }
+
+ private static <T> Class<T> readTypeClass(DataInputView in, ClassLoader userCodeClassLoader)
+ throws IOException {
+ return InstantiationUtil.resolveClassByName(in, userCodeClassLoader);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/CollectOutputAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/CollectOutputAdapter.java
new file mode 100644
index 000000000000..c1f70c2e1fa2
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/CollectOutputAdapter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.runtime.event.WatermarkEvent;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+import org.apache.flink.util.OutputTag;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Adapter clazz for {@code Output}.
+ */
+public class CollectOutputAdapter<T> implements Output<StreamRecord<T>> {
+
+ private final List<T> records;
+
+ public CollectOutputAdapter() {
+ this.records = new ArrayList<>();
+ }
+
+ public List<T> getRecords() {
+ return this.records;
+ }
+
+ @Override
+ public void emitWatermark(Watermark mark) {
+ // no operation
+ }
+
+ @Override
+ public void emitLatencyMarker(LatencyMarker latencyMarker) {
+ // no operation
+ }
+
+ @Override
+ public void emitRecordAttributes(RecordAttributes recordAttributes) {
+ // no operation
+ }
+
+ @Override
+ public void emitWatermark(WatermarkEvent watermarkEvent) {
+ // no operation
+ }
+
+ @Override
+ public void collect(StreamRecord<T> record) {
+ records.add(record.getValue());
+ }
+
+ @Override
+ public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
+ throw new UnsupportedOperationException("Side output not supported for CollectorOutput");
+ }
+
+ @Override
+ public void close() {
+ this.records.clear();
+ }
+
+ @Override
+ public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {
+ // no operation
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java
new file mode 100644
index 000000000000..2727b40b1393
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.types.variant.BinaryVariant;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * Adapter utils.
+ */
+public class DataTypeAdapterTestUtils {
+ public static void assertAsBinaryVariant(Object variantObject) {
+ assertInstanceOf(BinaryVariant.class, variantObject, "Variant column should be a BinaryVariant");
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java
new file mode 100644
index 000000000000..866e1ca8c6ae
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/ExecutionAttemptUtil.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+
+/**
+ * Adapter utils for execution attempt.
+ */
+public class ExecutionAttemptUtil {
+
+ public static ExecutionAttemptID randomId() {
+ return ExecutionAttemptID.randomId();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/KeyedStateStoreAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/KeyedStateStoreAdapter.java
new file mode 100644
index 000000000000..8c59e28c5ab5
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/KeyedStateStoreAdapter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.state.KeyedStateStore;
+
+import javax.annotation.Nonnull;
+
+public interface KeyedStateStoreAdapter extends KeyedStateStore {
+
+ @Override
+ default String getBackendTypeIdentifier() {
+ return "mock";
+ }
+
+ default <T> org.apache.flink.api.common.state.v2.ValueState<T> getValueState(@Nonnull org.apache.flink.api.common.state.v2.ValueStateDescriptor<T> valueStateDescriptor) {
+ throw new UnsupportedOperationException("getValueState() v2 is not supported.");
+ }
+
+ default <T> org.apache.flink.api.common.state.v2.ListState<T> getListState(@Nonnull org.apache.flink.api.common.state.v2.ListStateDescriptor<T> listStateDescriptor) {
+ throw new UnsupportedOperationException("getListState() v2 is not supported.");
+ }
+
+ default <K, V> org.apache.flink.api.common.state.v2.MapState<K, V> getMapState(@Nonnull org.apache.flink.api.common.state.v2.MapStateDescriptor<K, V> mapStateDescriptor) {
+ throw new UnsupportedOperationException("getMapState() v2 is not supported.");
+ }
+
+ default <T> org.apache.flink.api.common.state.v2.ReducingState<T> getReducingState(@Nonnull org.apache.flink.api.common.state.v2.ReducingStateDescriptor<T> reducingStateDescriptor) {
+ throw new UnsupportedOperationException("getReducingState() v2 is not supported.");
+ }
+
+ default <I, A, O> org.apache.flink.api.common.state.v2.AggregatingState<I, O> getAggregatingState(
+ @Nonnull org.apache.flink.api.common.state.v2.AggregatingStateDescriptor<I, A, O> aggregatingStateDescriptor) {
+ throw new UnsupportedOperationException("getAggregatingState() v2 is not supported.");
+ }
+
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/OperatorStateStoreAdapter.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/OperatorStateStoreAdapter.java
new file mode 100644
index 000000000000..03cd20e400a3
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/OperatorStateStoreAdapter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.state.BroadcastState;
+import org.apache.flink.api.common.state.OperatorStateStore;
+
+public interface OperatorStateStoreAdapter extends OperatorStateStore {
+
+ default <K, V> BroadcastState<K, V> getBroadcastState(org.apache.flink.api.common.state.v2.MapStateDescriptor<K, V> mapStateDescriptor) throws Exception {
+ throw new UnsupportedOperationException("getBroadcastState() v2 is not supported.");
+ }
+
+ default <S> org.apache.flink.api.common.state.v2.ListState<S> getListState(org.apache.flink.api.common.state.v2.ListStateDescriptor<S> listStateDescriptor) throws Exception {
+ throw new UnsupportedOperationException("getListState() v2 is not supported.");
+ }
+
+ default <S> org.apache.flink.api.common.state.v2.ListState<S> getUnionListState(org.apache.flink.api.common.state.v2.ListStateDescriptor<S> listStateDescriptor) throws Exception {
+ throw new UnsupportedOperationException("getUnionListState() v2 is not supported.");
+ }
+
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java
new file mode 100644
index 000000000000..a7a620b4ec13
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.adapter;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+
+/**
+ * StreamConfig for test goals.
+ */
+public class TestStreamConfigs {
+
+ public static void setupNetworkInputs(StreamConfig streamConfig, TypeSerializer<?>... inputSerializers) {
+ streamConfig.setupNetworkInputs(inputSerializers);
+ // Since Flink 1.16, need call serializeAllConfigs to serialize all object configs synchronously.
+ // See https://issues.apache.org/jira/browse/FLINK-26675.
+ streamConfig.serializeAllConfigs();
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/sink/utils/MockTaskInfo.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/sink/utils/MockTaskInfo.java
new file mode 100644
index 000000000000..2c3a353e411c
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/sink/utils/MockTaskInfo.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.sink.utils;
+
+import org.apache.flink.api.common.TaskInfo;
+
+/**
+ * Mock {@link TaskInfo} to use in tests.
+ */
+public class MockTaskInfo implements TaskInfo {
+
+ private final int numParallelSubtasks;
+ private final int subtaskIndex;
+ private int attemptNumber;
+
+ public MockTaskInfo(
+ int numParallelSubtasks,
+ int subtaskIndex,
+ int attemptNumber) {
+ this.numParallelSubtasks = numParallelSubtasks;
+ this.subtaskIndex = subtaskIndex;
+ this.attemptNumber = attemptNumber;
+ }
+
+ public void setAttemptNumber(int attemptNumber) {
+ this.attemptNumber = attemptNumber;
+ }
+
+ @Override
+ public String getTaskName() {
+ return "";
+ }
+
+ @Override
+ public int getMaxNumberOfParallelSubtasks() {
+ return numParallelSubtasks;
+ }
+
+ @Override
+ public int getIndexOfThisSubtask() {
+ return subtaskIndex;
+ }
+
+ @Override
+ public int getNumberOfParallelSubtasks() {
+ return numParallelSubtasks;
+ }
+
+ @Override
+ public int getAttemptNumber() {
+ return attemptNumber;
+ }
+
+ @Override
+ public String getTaskNameWithSubtasks() {
+ return getTaskName() + "_" + subtaskIndex;
+ }
+
+ @Override
+ public String getAllocationIDAsString() {
+ return "";
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
new file mode 100644
index 000000000000..aff1c32917cf
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+/**
+ * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector},
+ * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's
+ * nested-Parquet reader (FLINK-35702).
+ *
+ * <p>The accessors are wrappers over the existing public fields so legacy callers continue to
+ * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the
+ * Dremel-style read path is exercised end-to-end by integration tests in
+ * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes /
+ * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes).
+ */
+class TestHeapColumnVectorAccessors {
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapArrayVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapArrayVectorAccessorsReflectPublicFields() {
+ HeapIntVector child = new HeapIntVector(4);
+ HeapArrayVector vector = new HeapArrayVector(2, child);
+
+ long[] offsets = {0L, 2L};
+ long[] lengths = {2L, 2L};
+ HeapLongVector replacementChild = new HeapLongVector(4);
+
+ vector.setOffsets(offsets);
+ vector.setLengths(lengths);
+ vector.setChild(replacementChild);
+ vector.setSize(2);
+
+ assertArrayEquals(offsets, vector.getOffsets());
+ assertArrayEquals(lengths, vector.getLengths());
+ assertSame(replacementChild, vector.getChild());
+ assertEquals(2, vector.getSize());
+
+ // Backing public fields are kept in sync — preserves backward compatibility.
+ assertSame(offsets, vector.offsets);
+ assertSame(lengths, vector.lengths);
+ assertSame(replacementChild, vector.child);
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapMapColumnVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() {
+ HeapIntVector keys = new HeapIntVector(4);
+ HeapIntVector values = new HeapIntVector(4);
+
+ HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values);
+
+ assertEquals(3, vector.getOffsets().length);
+ assertEquals(3, vector.getLengths().length);
+ }
+
+ @Test
+ void heapMapColumnVectorAccessorsReflectInternalState() {
+ HeapIntVector keys = new HeapIntVector(4);
+ HeapIntVector values = new HeapIntVector(4);
+ HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values);
+
+ long[] offsets = {0L, 2L};
+ long[] lengths = {2L, 2L};
+ HeapLongVector newKeys = new HeapLongVector(4);
+ HeapLongVector newValues = new HeapLongVector(4);
+
+ vector.setOffsets(offsets);
+ vector.setLengths(lengths);
+ vector.setKeys(newKeys);
+ vector.setValues(newValues);
+ vector.setSize(2);
+
+ assertArrayEquals(offsets, vector.getOffsets());
+ assertArrayEquals(lengths, vector.getLengths());
+ assertSame(newKeys, vector.getKeys());
+ assertSame(newValues, vector.getValues());
+ // The Flink-2.1-style ColumnVector accessors return the same underlying child.
+ assertSame(newKeys, vector.getKeyColumnVector());
+ assertSame(newValues, vector.getValueColumnVector());
+ assertEquals(2, vector.getSize());
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapRowColumnVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() {
+ HeapIntVector intField = new HeapIntVector(2);
+ HeapLongVector longField = new HeapLongVector(2);
+ HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField);
+
+ WritableColumnVector[] originalFields = vector.getFields();
+ assertEquals(2, originalFields.length);
+ assertSame(intField, originalFields[0]);
+ assertSame(longField, originalFields[1]);
+ // Backing public field is kept in sync — preserves backward compatibility.
+ assertSame(originalFields, vector.vectors);
+
+ HeapIntVector replacement = new HeapIntVector(2);
+ WritableColumnVector[] replacementFields = {replacement, longField};
+ vector.setFields(replacementFields);
+
+ assertSame(replacementFields, vector.getFields());
+ assertSame(replacementFields, vector.vectors);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
new file mode 100644
index 000000000000..04f5809b9dac
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.columnar.vector.BytesColumnVector;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests for {@link ParquetDecimalVector}.
+ */
+public class TestParquetDecimalVector {
+
+ @Test
+ void testGetDecimalFromInt32Vector() {
+ // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.vector[0] = 12345;
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 5, 2);
+
+ assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromInt64Vector() {
+ // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true
+ HeapLongVector longVector = new HeapLongVector(1);
+ longVector.vector[0] = 1234567890123456L;
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 18, 4);
+
+ assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromBytesVectorAtLargePrecision() {
+ // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path
+ BigDecimal original = new BigDecimal("12345678901234567890.1234567890");
+ byte[] unscaled = original.unscaledValue().toByteArray();
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ bytesVector.appendBytes(0, unscaled, 0, unscaled.length);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 30, 10);
+
+ assertEquals(original, decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromBytesVectorAtSmallPrecision() {
+ // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the
+ // dispatch must fall through to the bytes branch rather than require an IntColumnVector.
+ BigDecimal original = new BigDecimal("123.45");
+ byte[] unscaled = original.unscaledValue().toByteArray();
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ bytesVector.appendBytes(0, unscaled, 0, unscaled.length);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 5, 2);
+
+ assertEquals(original, decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalThrowsOnUnsupportedVectorType() {
+ // A large-precision request must have a bytes-backed child; any other writable child is an
+ // illegal combination and must be surfaced via Preconditions.checkArgument.
+ ColumnVector unsupported = new HeapShortVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported);
+
+ assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10));
+ }
+
+ @Test
+ void testIsNullAtDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ intVector.vector[0] = 1;
+ intVector.setNullAt(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ assertFalse(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+
+ @Test
+ void testWritableIntRoundTrip() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.setInt(0, 42);
+
+ assertEquals(42, wrapped.getInt(0));
+ assertEquals(42, intVector.vector[0]);
+ }
+
+ @Test
+ void testWritableLongRoundTrip() {
+ HeapLongVector longVector = new HeapLongVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector);
+
+ wrapped.setLong(0, 9876543210L);
+
+ assertEquals(9876543210L, wrapped.getLong(0));
+ assertEquals(9876543210L, longVector.vector[0]);
+ }
+
+ @Test
+ void testWritableBytesRoundTrip() {
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+ byte[] payload = new byte[] {0x01, 0x02, 0x03};
+
+ wrapped.appendBytes(0, payload, 0, payload.length);
+
+ BytesColumnVector.Bytes out = wrapped.getBytes(0);
+ assertEquals(payload.length, out.len);
+ assertEquals(0x01, out.data[out.offset]);
+ assertEquals(0x02, out.data[out.offset + 1]);
+ assertEquals(0x03, out.data[out.offset + 2]);
+ }
+
+ @Test
+ void testResetDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.setNullAt(0);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+ assertTrue(wrapped.isNullAt(0));
+
+ wrapped.reset();
+
+ assertFalse(wrapped.isNullAt(0));
+ }
+
+ @Test
+ void testFillWithNullsDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.fillWithNulls();
+
+ assertTrue(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+
+ @Test
+ void testSetNullAtDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.setNullAt(0);
+ wrapped.setNulls(1, 1);
+
+ assertTrue(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
new file mode 100644
index 000000000000..ea222dad576a
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.TimestampData;
+import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when
+ * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702).
+ *
+ * <p>The factory is exercised end-to-end by integration tests through
+ * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece
+ * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each
+ * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation}
+ * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values
+ * using both the values-reader and dictionary code paths.
+ */
+class TestParquetDataColumnReaderFactory {
+
+ // -----------------------------------------------------------------------------------------------
+ // Type dispatch
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void valuesReaderDispatchInt96TimestampUsesInt96Reader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(OriginalType.TIMESTAMP_MILLIS)
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(OriginalType.TIMESTAMP_MICROS)
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt32DoesNotUseTimestampReader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader);
+ }
+
+ @Test
+ void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ type, new StubDictionary(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches)
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ long epochMillis = 1_700_000_000_123L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochMillis), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochMillis, ts.getMillisecond());
+ assertEquals(0, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS))
+ .named("ts");
+ long epochMicros = 1_700_000_000_123_456L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochMicros), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochMicros / 1_000L, ts.getMillisecond());
+ // 456 microseconds remain → 456_000 nanoseconds within the millisecond
+ assertEquals(456_000, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("ts");
+ long epochNanos = 1_700_000_000_123_456_789L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochNanos), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochNanos / 1_000_000L, ts.getMillisecond());
+ assertEquals(456_789, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampMillisFromDictionaryInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ long epochMillis = 1_700_000_000_456L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ type, new StubDictionary(epochMillis), true);
+
+ TimestampData ts = reader.readTimestamp(0);
+ assertNotNull(ts);
+ assertEquals(epochMillis, ts.getMillisecond());
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // Stubs (only the methods exercised by the dispatch + decoding tests above)
+ // -----------------------------------------------------------------------------------------------
+
+ /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */
+ private static final class StubValuesReader extends ValuesReader {
+ private final long fixedLong;
+
+ StubValuesReader() {
+ this(0L);
+ }
+
+ StubValuesReader(long fixedLong) {
+ this.fixedLong = fixedLong;
+ }
+
+ @Override
+ public long readLong() {
+ return fixedLong;
+ }
+
+ @Override
+ public void skip() {
+ // unused
+ }
+ }
+
+ /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */
+ private static final class StubDictionary extends Dictionary {
+ private final long fixedLong;
+
+ StubDictionary() {
+ this(0L);
+ }
+
+ StubDictionary(long fixedLong) {
+ super(null);
+ this.fixedLong = fixedLong;
+ }
+
+ @Override
+ public Binary decodeToBinary(int id) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long decodeToLong(int id) {
+ return fixedLong;
+ }
+
+ @Override
+ public int getMaxId() {
+ return 0;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
new file mode 100644
index 000000000000..2b71bae4dc15
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Tests for {@link ParquetGroupField}.
+ */
+public class TestParquetGroupField {
+
+ @Test
+ void testChildrenWithAllNonNullEntriesAreRetained() {
+ ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1);
+ List<ParquetField> children = Arrays.asList(c0, c1);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children);
+
+ assertEquals(2, group.getChildren().size());
+ assertSame(c0, group.getChildren().get(0));
+ assertSame(c1, group.getChildren().get(1));
+ }
+
+ @Test
+ void testChildrenMayContainNullForSchemaEvolution() {
+ // A ROW field present in the requested Flink schema but absent from the Parquet file is
+ // represented by a null slot in `children`. The group must allow this (Hudi-specific
+ // extension over Flink's ImmutableList-backed equivalent).
+ ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ List<ParquetField> children = Arrays.asList(present, null);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children);
+
+ assertEquals(2, group.getChildren().size());
+ assertNotNull(group.getChildren().get(0));
+ assertNull(group.getChildren().get(1));
+ }
+
+ @Test
+ void testChildrenListIsUnmodifiable() {
+ ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child));
+
+ assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null));
+ assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0));
+ }
+
+ @Test
+ void testChildrenListIsDefensivelyCopied() {
+ // Mutations to the caller-supplied list must not be visible through the group.
+ ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ List<ParquetField> mutable = new ArrayList<>();
+ mutable.add(child);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable);
+ mutable.add(null);
+
+ assertEquals(1, group.getChildren().size());
+ }
+
+ @Test
+ void testNullChildrenListThrows() {
+ assertThrows(
+ NullPointerException.class,
+ () -> new ParquetGroupField(rowType(), 0, 1, true, null));
+ }
+
+ @Test
+ void testEmptyChildrenListIsAllowed() {
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList());
+
+ assertEquals(0, group.getChildren().size());
+ }
+
+ @Test
+ void testFieldMetadataIsExposed() {
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList());
+
+ assertEquals(2, group.getRepetitionLevel());
+ assertEquals(5, group.getDefinitionLevel());
+ assertFalse(group.isRequired());
+ }
+
+ private static LogicalType rowType() {
+ return RowType.of(new IntType());
+ }
+
+ private static ColumnDescriptor descriptor() {
+ PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f");
+ return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0);
+ }
+}
diff --git a/hudi-lakehouse/scripts/build-jars.sh b/hudi-lakehouse/scripts/build-jars.sh
index 0c531bb861c5..0243146ce2f7 100755
--- a/hudi-lakehouse/scripts/build-jars.sh
+++ b/hudi-lakehouse/scripts/build-jars.sh
@@ -43,7 +43,7 @@ if [[ "$BUILD_BUNDLE" == 1 ]]; then
# disables Maven's activeByDefault profiles, so relying on defaults is fragile.
echo ">>> Building Hudi Spark bundle (spark${SPARK_VERSION})"
(cd "$REPO_ROOT" && mvn clean package -pl packaging/hudi-spark-bundle -am \
- "-Dspark${SPARK_VERSION}" -Dflink2.1 "${MVN_ARGS[@]}")
+ "-Dspark${SPARK_VERSION}" -Dflink2.2 "${MVN_ARGS[@]}")
fi
if [[ "$BUILD_PLUGIN" == 1 ]]; then
diff --git a/packaging/bundle-validation/base/build_flink221hive313spark351.sh b/packaging/bundle-validation/base/build_flink221hive313spark351.sh
new file mode 100755
index 000000000000..c611c84ea6c7
--- /dev/null
+++ b/packaging/bundle-validation/base/build_flink221hive313spark351.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+docker build \
+ --build-arg HIVE_VERSION=3.1.3 \
+ --build-arg FLINK_VERSION=2.2.1 \
+ --build-arg SPARK_VERSION=3.5.1 \
+ --build-arg SPARK_HADOOP_VERSION=3 \
+ --build-arg HADOOP_VERSION=3.3.5 \
+ -t hudi-ci-bundle-validation-base:flink221hive313spark351 .
+docker image tag hudi-ci-bundle-validation-base:flink221hive313spark351 apachehudi/hudi-ci-bundle-validation-base:flink221hive313spark351
diff --git a/packaging/bundle-validation/ci_run.sh b/packaging/bundle-validation/ci_run.sh
index 178b647dd033..922ba0af2030 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -108,6 +108,9 @@ elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.12' ]];
elif [[ ${FLINK_PROFILE} == 'flink2.1' ]]; then
IMAGE_TAG=flink211hive313spark351
FLINK_VERSION=2.1.1
+ elif [[ ${FLINK_PROFILE} == 'flink2.2' ]]; then
+ IMAGE_TAG=flink221hive313spark351
+ FLINK_VERSION=2.2.1
else
echo "Unsupported Flink profile ${FLINK_PROFILE}"
exit 1
@@ -236,6 +239,8 @@ else
HUDI_FLINK_BUNDLE_NAME=hudi-flink2.0-bundle
elif [[ ${FLINK_PROFILE} == 'flink2.1' ]]; then
HUDI_FLINK_BUNDLE_NAME=hudi-flink2.1-bundle
+ elif [[ ${FLINK_PROFILE} == 'flink2.2' ]]; then
+ HUDI_FLINK_BUNDLE_NAME=hudi-flink2.2-bundle
fi
echo "Downloading bundle jars from base URL - $REPO_BASE_URL ..."
diff --git a/packaging/bundle-validation/validate.sh b/packaging/bundle-validation/validate.sh
index f2b05325e567..5a893cd225c0 100755
--- a/packaging/bundle-validation/validate.sh
+++ b/packaging/bundle-validation/validate.sh
@@ -360,8 +360,8 @@ test_cli_bundle() {
############################
# run flink bundle test only for flink2.x case, since there is problem for java 11 and spark3.5 bundle (HUDI-8608).
-if [[ "${FLINK_HOME}" == *"2.0"* || "${FLINK_HOME}" == *"2.1"* ]]; then
- echo "::warning::validate.sh validating flink 2.0 bundle"
+if [[ "${FLINK_HOME}" == *"flink-2."* ]]; then
+ echo "::warning::validate.sh validating flink 2.x bundle"
test_flink_bundle
if [ "$?" -ne 0 ]; then
exit 1
diff --git a/pom.xml b/pom.xml
index 5bb7060cc8ba..53ab25ea108a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -162,19 +162,19 @@
<spark3.version>${spark35.version}</spark3.version>
<spark4.version>4.0.2</spark4.version>
<sparkbundle.version></sparkbundle.version>
+ <flink2.2.version>2.2.1</flink2.2.version>
<flink2.1.version>2.1.1</flink2.1.version>
<flink2.0.version>2.0.0</flink2.0.version>
<flink1.20.version>1.20.1</flink1.20.version>
<flink1.19.version>1.19.2</flink1.19.version>
<flink1.18.version>1.18.1</flink1.18.version>
- <flink.version>${flink2.1.version}</flink.version>
- <hudi.flink.module>hudi-flink2.1.x</hudi.flink.module>
- <flink.bundle.version>2.1</flink.bundle.version>
+ <flink.version>${flink2.2.version}</flink.version>
+ <hudi.flink.module>hudi-flink2.2.x</hudi.flink.module>
+ <flink.bundle.version>2.2</flink.bundle.version>
<!-- This is fixed to match with version from flink-avro -->
<flink.avro.version>1.11.4</flink.avro.version>
<flink.format.parquet.version>1.15.2</flink.format.parquet.version>
- <!-- check kafka version -->
- <flink.connector.kafka.version>4.0.1-2.0</flink.connector.kafka.version>
+ <flink.connector.kafka.version>5.0.0-2.2</flink.connector.kafka.version>
<flink.runtime.artifactId>flink-runtime</flink.runtime.artifactId>
<flink.table.runtime.artifactId>flink-table-runtime</flink.table.runtime.artifactId>
<flink.table.planner.artifactId>flink-table-planner_2.12</flink.table.planner.artifactId>
@@ -3080,6 +3080,29 @@
</activation>
</profile>
+ <profile>
+ <id>flink2.2</id>
+ <properties>
+ <flink.version>${flink2.2.version}</flink.version>
+ <hudi.flink.module>hudi-flink2.2.x</hudi.flink.module>
+ <flink.bundle.version>2.2</flink.bundle.version>
+ <flink.connector.kafka.version>5.0.0-2.2</flink.connector.kafka.version>
+ <orc.flink.version>1.5.6</orc.flink.version>
+ <flink.avro.version>1.11.4</flink.avro.version>
+ <flink.format.parquet.version>1.15.2</flink.format.parquet.version>
+ <flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility</flink.hadoop.compatibility.artifactId>
+ </properties>
+ <modules>
+ <module>hudi-flink-datasource/hudi-flink2.2.x</module>
+ </modules>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>flink2.2</name>
+ </property>
+ </activation>
+ </profile>
+
<profile>
<id>flink2.1</id>
<properties>
@@ -3097,7 +3120,6 @@
<module>hudi-flink-datasource/hudi-flink2.1.x</module>
</modules>
<activation>
- <activeByDefault>true</activeByDefault>
<property>
<name>flink2.1</name>
</property>
diff --git a/scripts/release/deploy_staging_jars.sh b/scripts/release/deploy_staging_jars.sh
index 30b4000238ea..a5947e9a4757 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -78,6 +78,7 @@ declare -a ALL_VERSION_OPTS=(
"-T 1C -Dscala-2.12 -Dspark3.5 -Dflink1.20 -Davro.version=1.11.4 -Dparquet.version=1.13.1 -pl packaging/hudi-flink-bundle -am"
"-T 1C -Dscala-2.12 -Dspark3.5 -Dflink2.0 -Davro.version=1.11.4 -Dparquet.version=1.14.4 -pl packaging/hudi-flink-bundle -am"
"-T 1C -Dscala-2.12 -Dspark3.5 -Dflink2.1 -Davro.version=1.11.4 -Dparquet.version=1.15.2 -pl packaging/hudi-flink-bundle -am"
+"-T 1C -Dscala-2.12 -Dspark3.5 -Dflink2.2 -Davro.version=1.11.4 -Dparquet.version=1.15.2 -pl packaging/hudi-flink-bundle -am"
)
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
diff --git a/scripts/release/validate_staged_bundles.sh b/scripts/release/validate_staged_bundles.sh
index 21135a7863c1..f3f4e3aaaf11 100755
--- a/scripts/release/validate_staged_bundles.sh
+++ b/scripts/release/validate_staged_bundles.sh
@@ -86,7 +86,7 @@ declare -a extensions=("-javadoc.jar" "-javadoc.jar.asc" "-javadoc.jar.md5" "-ja
# hudi-trino is a plain library jar, not a bundle, but it is staged and validated like the bundles.
declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle" "hudi-cli-bundle_2.12" "hudi-cli-bundle_2.13" "hudi-datahub-sync-bundle"
"hudi-flink1.18-bundle" "hudi-flink1.19-bundle" "hudi-flink1.20-bundle"
-"hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle"
+"hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-flink2.2-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle"
"hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle" "hudi-presto-bundle"
"hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12"
"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13" "hudi-spark4.1-bundle_2.13" "hudi-spark4.2-bundle_2.13" "hudi-timeline-server-bundle"