You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/08/05 07:35:18 UTC
(hudi) branch release-1.2.1 updated (dbfafbcf5738 -> fbb5c1f1d90e)
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from dbfafbcf5738 feat(spark): add restore_to_instant stored procedure (#18696)
new 8f74fdaf2f52 feat(spark): add show_inflight_commits and cleanup_stale_inflight_com… (#18709)
new 88b4e85809b1 chore: add spark4.1 and flink2.1 profile entries to RC bundle validation (#18796)
new 18cddb5b6d86 fix(flink): fix disable table service not effective in hudi-flink (#13875)
new a15bc7bd8827 feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (#18809)
new 532edf360a4b chore: Fix stale zip file for variant backward compatibility test (#18815)
new 9c3137ae3c45 chore: Add release information for Apache Hudi 1.2.0 (#18831)
new c7be772580da fix(spark): Add options for archive procedure (#18437)
new 4d06094008a7 fix: RLI bootstrap fails due to NPE with cleaner table service (#18836)
new 3e3d0b8af1ee chore: Fix Flink CI Maven profile arguments (#18845)
new d24332afdce4 [MINOR] Handle cancellation error with HoodieMetadataTableValidator (#18371)
new 6834874bedfa docs: Update DOAP file to include 0.15.1 (#18838)
new fbb5c1f1d90e feat(trino): [RFC-105] Trino Hudi Connector — Shim/Bundle Refactor (#18782)
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/bot.yml | 8 +-
.github/workflows/release_candidate_validation.yml | 8 +
doap_HUDI.rdf | 16 +
.../org/apache/hudi/cli/commands/SparkMain.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 24 +-
.../TestHoodieBackedTableMetadataWriter.java | 45 ++
...adataIndexException.java => ExceptionUtil.java} | 31 +-
.../apache/hudi/exception/TestExceptionUtil.java | 57 ++
.../variant_backward_compat/variant_mor_avro.zip | Bin 87511 -> 104069 bytes
.../apache/hudi/configuration/FlinkOptions.java | 6 +
.../apache/hudi/configuration/OptionsResolver.java | 36 +-
.../org/apache/hudi/sink/v2/utils/PipelinesV2.java | 8 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 6 +-
.../org/apache/hudi/table/HoodieTableSink.java | 6 +-
.../hudi/configuration/TestOptionsResolver.java | 68 ++
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 29 +
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 31 +
.../table/format/cow/utils/NestedPositionUtil.java | 4 +-
.../reader/ParquetDataColumnReaderFactory.java | 8 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 688 ++++++++++++---------
.../table/format/cow/utils/BooleanArrayList.java | 0
.../hudi/table/format/cow/utils/IntArrayList.java | 0
.../hudi/table/format/cow/utils/LongArrayList.java | 0
.../table/format/cow/utils/NestedPositionUtil.java | 4 +-
.../format/cow/vector/ColumnarGroupArrayData.java | 179 ------
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 -----
.../cow/vector/HeapArrayGroupColumnVector.java | 53 --
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 75 ++-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++++-
.../cow/vector/position/CollectionPosition.java | 0
.../cow/vector/position/LevelDelegation.java | 0
.../format/cow/vector/position/RowPosition.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 473 --------------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 --
.../format/cow/vector/reader/MapColumnReader.java | 56 --
.../cow/vector/reader/NestedColumnReader.java | 0
.../vector/reader/NestedPrimitiveColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 +++-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 0
.../format/cow/vector/type/ParquetGroupField.java | 0
.../cow/vector/type/ParquetPrimitiveField.java | 0
.../cow/vector/TestHeapColumnVectorAccessors.java | 0
.../cow/vector/TestParquetDecimalVector.java | 0
.../reader/TestParquetDataColumnReaderFactory.java | 0
.../cow/vector/type/TestParquetGroupField.java | 0
.../org/apache/hudi/HoodieTimelineCleanupUtil.java | 59 ++
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 60 +-
.../scala/org/apache/hudi/TestHoodieCLIUtils.scala | 104 ++++
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 17 +-
.../procedures/ArchiveCommitsProcedure.scala | 99 ++-
.../CleanupStaleInflightCommitsProcedure.scala | 224 +++++++
.../hudi/command/procedures/HoodieProcedures.scala | 2 +
.../procedures/ShowInflightCommitsProcedure.scala | 113 ++++
.../procedure/TestArchiveCommitsProcedure.scala | 147 +++--
.../TestCleanupStaleInflightCommitsProcedure.scala | 413 +++++++++++++
.../TestShowInflightCommitsProcedure.scala | 151 +++++
.../utilities/HoodieMetadataTableValidator.java | 3 +
rfc/rfc-105/rfc-105.md | 225 +++++++
63 files changed, 2769 insertions(+), 1467 deletions(-)
copy hudi-common/src/main/java/org/apache/hudi/exception/{HoodieMetadataIndexException.java => ExceptionUtil.java} (56%)
create mode 100644 hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java (98%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java (100%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java (100%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java (100%)
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala
create mode 100644 rfc/rfc-105/rfc-105.md
(hudi) 10/12: [MINOR] Handle cancellation error with HoodieMetadataTableValidator (#18371)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d24332afdce4d0ad2c4648cfbe80d516e01c034d
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Tue May 26 20:48:15 2026 +0530
[MINOR] Handle cancellation error with HoodieMetadataTableValidator (#18371)
---------
Co-authored-by: Lokesh Jain <lj...@Lokeshs-MacBook-Pro.local>
(cherry picked from commit 516d9e2093a5a10fa98d9e446675e66039db6647)
---
.../org/apache/hudi/exception/ExceptionUtil.java | 51 +++++++++++++++++++
.../apache/hudi/exception/TestExceptionUtil.java | 57 ++++++++++++++++++++++
.../utilities/HoodieMetadataTableValidator.java | 3 ++
3 files changed, 111 insertions(+)
diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java
new file mode 100644
index 000000000000..92e2e3cc5356
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.exception;
+
+import org.apache.hudi.common.util.StringUtils;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Util class for exception analysis.
+ */
+public final class ExceptionUtil {
+ private ExceptionUtil() {
+ }
+
+ /**
+ * Returns true if error message is contained in any nested exception of provided {@link Throwable}.
+ */
+ public static boolean validateErrorMsg(@Nonnull Throwable t, String errorMsg) {
+ if (StringUtils.isNullOrEmpty(errorMsg)) {
+ return false;
+ }
+
+ Throwable cause = t;
+ while (cause != null) {
+ if (cause.getMessage() != null && cause.getMessage().contains(errorMsg)) {
+ return true;
+ }
+ cause = cause.getCause();
+ }
+
+ return false;
+ }
+}
diff --git a/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java
new file mode 100644
index 000000000000..dba850b1b907
--- /dev/null
+++ b/hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.exception;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.apache.hudi.exception.ExceptionUtil.validateErrorMsg;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class TestExceptionUtil {
+
+ @Test
+ void testValidateErrorMsgInNestedCause() {
+ IOException rootException = new IOException("File not found: data.parquet");
+ IllegalArgumentException middleException = new IllegalArgumentException("Invalid argument", rootException);
+ RuntimeException topException = new RuntimeException("Operation failed", middleException);
+
+ // Check top level exception msg
+ assertTrue(validateErrorMsg(topException, "File not found"));
+ assertTrue(validateErrorMsg(topException, "data.parquet"));
+ // Check nested exception msg
+ assertTrue(validateErrorMsg(topException, "Invalid argument"));
+ assertTrue(validateErrorMsg(topException, "Operation failed"));
+ // Validate no exception matches
+ assertFalse(validateErrorMsg(topException, "Connection refused"));
+ }
+
+ @Test
+ void testValidateErrorMsgWithEmptyMessage() {
+ RuntimeException exceptionWithMessage = new RuntimeException("Some error");
+ assertFalse(validateErrorMsg(exceptionWithMessage, ""));
+
+ RuntimeException exceptionWithoutMessage = new RuntimeException();
+ // Empty string should not be found in any message (including null)
+ assertFalse(validateErrorMsg(exceptionWithoutMessage, ""));
+ }
+}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index 778cd0090bc7..daf9370c3550 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -75,6 +75,7 @@ import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.data.HoodieSparkRDDUtils;
+import org.apache.hudi.exception.ExceptionUtil;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieValidationException;
@@ -690,6 +691,8 @@ public class HoodieMetadataTableValidator implements Serializable {
} catch (SparkException sparkException) {
if (sparkException.getCause() instanceof HoodieValidationException) {
throw (HoodieValidationException) sparkException.getCause();
+ } else if (ExceptionUtil.validateErrorMsg(sparkException, "cancelled because SparkContext was shut down")) {
+ throw new HoodieException(sparkException);
} else {
throw new HoodieValidationException("Unexpected spark failure", sparkException);
}
(hudi) 03/12: fix(flink): fix disable table service not effective in hudi-flink (#13875)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 18cddb5b6d866145bd5ee150e49187cb4b7e2938
Author: fhan <aa...@gmail.com>
AuthorDate: Thu May 21 16:32:51 2026 +0800
fix(flink): fix disable table service not effective in hudi-flink (#13875)
make the HoodieWriteConfig.TABLE_SERVICES_ENABLED effective for Flink.
---------
Co-authored-by: fhan <yf...@jd.com>
Co-authored-by: danny0405 <yu...@gmail.com>
(cherry picked from commit 50eb95c7d6f5da986244e53627e54a703f6f8678)
---
.../apache/hudi/configuration/FlinkOptions.java | 6 ++
.../apache/hudi/configuration/OptionsResolver.java | 36 ++++++++++--
.../org/apache/hudi/sink/v2/utils/PipelinesV2.java | 8 ++-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 6 +-
.../org/apache/hudi/table/HoodieTableSink.java | 6 +-
.../hudi/configuration/TestOptionsResolver.java | 68 ++++++++++++++++++++++
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 29 +++++++++
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 31 ++++++++++
8 files changed, 178 insertions(+), 12 deletions(-)
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
index 4910c721e09c..fbe29f93ef5b 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
@@ -188,6 +188,12 @@ public class FlinkOptions extends HoodieConfig {
.defaultValue(false) // keep sync with hoodie style
.withDescription("If enabled, the checkpoint Id will also be written to hudi metadata.");
+ public static final ConfigOption<Boolean> TABLE_SERVICES_ENABLED = ConfigOptions
+ .key(HoodieWriteConfig.TABLE_SERVICES_ENABLED.key())
+ .booleanType()
+ .defaultValue(HoodieWriteConfig.TABLE_SERVICES_ENABLED.defaultValue())
+ .withDescription("Master control to disable all table services including archive, clean, compact, cluster, etc.");
+
// ------------------------------------------------------------------------
// Changelog Capture Options
// ------------------------------------------------------------------------
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java
index 628474bde7d6..74c5257cf44f 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsResolver.java
@@ -292,7 +292,7 @@ public class OptionsResolver {
* @param conf The flink configuration.
*/
public static boolean needsAsyncCompaction(Configuration conf) {
- return OptionsResolver.isMorTable(conf) && conf.get(FlinkOptions.COMPACTION_ASYNC_ENABLED);
+ return OptionsResolver.isMorTable(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.COMPACTION_ASYNC_ENABLED);
}
/**
@@ -301,7 +301,7 @@ public class OptionsResolver {
* @param conf The flink configuration.
*/
public static boolean needsAsyncMetadataCompaction(Configuration conf) {
- return isStreamingIndexWriteEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_ASYNC_ENABLED);
+ return isStreamingIndexWriteEnabled(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_ASYNC_ENABLED);
}
/**
@@ -310,7 +310,7 @@ public class OptionsResolver {
* @param conf The flink configuration.
*/
public static boolean needsScheduleMdtCompaction(Configuration conf) {
- return isStreamingIndexWriteEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_SCHEDULE_ENABLED);
+ return isStreamingIndexWriteEnabled(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.METADATA_COMPACTION_SCHEDULE_ENABLED);
}
/**
@@ -320,7 +320,9 @@ public class OptionsResolver {
*/
public static boolean needsScheduleCompaction(Configuration conf) {
return OptionsResolver.isMorTable(conf)
- && conf.get(FlinkOptions.COMPACTION_SCHEDULE_ENABLED) && !isAppendMode(conf);
+ && areTableServicesEnabled(conf)
+ && conf.get(FlinkOptions.COMPACTION_SCHEDULE_ENABLED)
+ && !isAppendMode(conf);
}
/**
@@ -329,7 +331,7 @@ public class OptionsResolver {
* @param conf The flink configuration.
*/
public static boolean needsAsyncClustering(Configuration conf) {
- return isInsertOperation(conf) && conf.get(FlinkOptions.CLUSTERING_ASYNC_ENABLED);
+ return isInsertOperation(conf) && areTableServicesEnabled(conf) && conf.get(FlinkOptions.CLUSTERING_ASYNC_ENABLED);
}
/**
@@ -338,6 +340,9 @@ public class OptionsResolver {
* @param conf The flink configuration.
*/
public static boolean needsScheduleClustering(Configuration conf) {
+ if (!areTableServicesEnabled(conf)) {
+ return false;
+ }
if (!conf.get(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED)) {
return false;
}
@@ -544,6 +549,20 @@ public class OptionsResolver {
return WriteConcurrencyMode.isNonBlockingConcurrencyControl(config.getString(HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key(), HoodieWriteConfig.WRITE_CONCURRENCY_MODE.defaultValue()));
}
+ /**
+ * Returns whether the cleaning for failed writes is enabled as lazy.
+ */
+ public static boolean isLazyFailedWritesCleaning(Configuration conf) {
+ return needsAsyncCleaning(conf) && isLazyFailedWritesCleanPolicy(conf);
+ }
+
+ /**
+ * Returns whether there is need for async cleaning (planning & execution).
+ */
+ public static boolean needsAsyncCleaning(Configuration conf) {
+ return areTableServicesEnabled(conf);
+ }
+
/**
* Returns whether Cleaner's failed writes policy is set to lazy
*/
@@ -574,6 +593,13 @@ public class OptionsResolver {
return (isCowTable(conf) || conf.get(FlinkOptions.CDC_ENABLED)) && isUpsertOperation(conf);
}
+ /**
+ * Returns whether table services are enabled.
+ */
+ public static boolean areTableServicesEnabled(Configuration conf) {
+ return conf.get(FlinkOptions.TABLE_SERVICES_ENABLED);
+ }
+
/**
* Returns the customized insert partitioner instance.
*/
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java
index 6e93c02bc137..107e8673c42b 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/v2/utils/PipelinesV2.java
@@ -119,7 +119,7 @@ public class PipelinesV2 {
DataStream<RowData> pipeline = Pipelines.append(conf, rowType, dataStream);
if (OptionsResolver.needsAsyncClustering(conf)) {
return clusterV2(conf, rowType, pipeline);
- } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) {
+ } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) {
// add clean function to rollback failed writes for lazy failed writes cleaning policy
return cleanV2(conf, pipeline);
} else {
@@ -138,8 +138,10 @@ public class PipelinesV2 {
conf.set(FlinkOptions.COMPACTION_OPERATION_EXECUTE_ASYNC_ENABLED, false);
}
return compactV2(conf, pipeline);
- } else {
+ } else if (OptionsResolver.needsAsyncCleaning(conf)) {
return cleanV2(conf, pipeline);
+ } else {
+ return pipeline;
}
}
@@ -156,7 +158,7 @@ public class PipelinesV2 {
if (OptionsResolver.isBulkInsertOperation(conf)) {
return conf.get(FlinkOptions.WRITE_TASKS);
} else if (OptionsResolver.isAppendMode(conf)) {
- return OptionsResolver.needsAsyncClustering(conf) || OptionsResolver.isLazyFailedWritesCleanPolicy(conf)
+ return OptionsResolver.needsAsyncClustering(conf) || OptionsResolver.isLazyFailedWritesCleaning(conf)
? 1 : conf.get(FlinkOptions.WRITE_TASKS);
} else {
return 1;
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java
index 721b3f94e470..58be515ca901 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java
@@ -97,7 +97,7 @@ public class HoodieFlinkStreamer {
pipeline = Pipelines.append(conf, rowType, dataStream);
if (OptionsResolver.needsAsyncClustering(conf)) {
Pipelines.cluster(conf, rowType, pipeline);
- } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) {
+ } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) {
// add clean function to rollback failed writes for lazy failed writes cleaning policy
Pipelines.clean(conf, pipeline);
} else {
@@ -108,8 +108,10 @@ public class HoodieFlinkStreamer {
pipeline = Pipelines.hoodieStreamWrite(conf, rowType, hoodieRecordDataStream);
if (OptionsResolver.needsAsyncCompaction(conf)) {
Pipelines.compact(conf, pipeline);
- } else {
+ } else if (OptionsResolver.needsAsyncCleaning(conf)) {
Pipelines.clean(conf, pipeline);
+ } else {
+ Pipelines.dummySink(pipeline);
}
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
index f46e2e672222..45bfde998af6 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
@@ -112,7 +112,7 @@ public class HoodieTableSink implements
DataStream<RowData> pipeline = Pipelines.append(conf, rowType, dataStream);
if (OptionsResolver.needsAsyncClustering(conf)) {
return Pipelines.cluster(conf, rowType, pipeline);
- } else if (OptionsResolver.isLazyFailedWritesCleanPolicy(conf)) {
+ } else if (OptionsResolver.isLazyFailedWritesCleaning(conf)) {
// add clean function to rollback failed writes for lazy failed writes cleaning policy
return Pipelines.clean(conf, pipeline);
} else {
@@ -131,8 +131,10 @@ public class HoodieTableSink implements
conf.set(FlinkOptions.COMPACTION_OPERATION_EXECUTE_ASYNC_ENABLED, false);
}
return Pipelines.compact(conf, pipeline);
- } else {
+ } else if (OptionsResolver.needsAsyncCleaning(conf)) {
return Pipelines.clean(conf, pipeline);
+ } else {
+ return Pipelines.dummySink(pipeline);
}
};
}
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java
index 3d9a6bba9660..aef93b7d10fb 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsResolver.java
@@ -19,6 +19,7 @@
package org.apache.hudi.configuration;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
+import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.config.HoodieCleanConfig;
@@ -123,4 +124,71 @@ public class TestOptionsResolver {
conf.set(FlinkOptions.PATH, tempFile.getAbsolutePath());
return conf;
}
+
+ @Test
+ void testAreTableServicesEnabled() {
+ Configuration conf = new Configuration();
+ // default value should be true
+ assertTrue(OptionsResolver.areTableServicesEnabled(conf));
+
+ // explicitly set to true
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, true);
+ assertTrue(OptionsResolver.areTableServicesEnabled(conf));
+
+ // explicitly set to false
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+ assertFalse(OptionsResolver.areTableServicesEnabled(conf));
+ }
+
+ @Test
+ void testTableServicesGateCompactionAndCleaning() {
+ Configuration conf = getConf();
+ conf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name());
+ conf.setString(HoodieCleanConfig.FAILED_WRITES_CLEANER_POLICY.key(), HoodieFailedWritesCleaningPolicy.LAZY.name());
+
+ assertTrue(OptionsResolver.needsAsyncCompaction(conf));
+ assertTrue(OptionsResolver.needsScheduleCompaction(conf));
+ assertTrue(OptionsResolver.needsAsyncCleaning(conf));
+ assertTrue(OptionsResolver.isLazyFailedWritesCleanPolicy(conf));
+ assertTrue(OptionsResolver.isLazyFailedWritesCleaning(conf));
+
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+
+ assertFalse(OptionsResolver.needsAsyncCompaction(conf));
+ assertFalse(OptionsResolver.needsScheduleCompaction(conf));
+ assertFalse(OptionsResolver.needsAsyncCleaning(conf));
+ assertTrue(OptionsResolver.isLazyFailedWritesCleanPolicy(conf));
+ assertFalse(OptionsResolver.isLazyFailedWritesCleaning(conf));
+ }
+
+ @Test
+ void testTableServicesGateMetadataCompaction() {
+ Configuration conf = getConf();
+ conf.set(FlinkOptions.METADATA_ENABLED, true);
+ conf.set(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.RECORD_LEVEL_INDEX.name());
+
+ assertTrue(OptionsResolver.needsAsyncMetadataCompaction(conf));
+ assertTrue(OptionsResolver.needsScheduleMdtCompaction(conf));
+
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+
+ assertFalse(OptionsResolver.needsAsyncMetadataCompaction(conf));
+ assertFalse(OptionsResolver.needsScheduleMdtCompaction(conf));
+ }
+
+ @Test
+ void testTableServicesGateClustering() {
+ Configuration conf = getConf();
+ conf.set(FlinkOptions.OPERATION, WriteOperationType.INSERT.value());
+ conf.set(FlinkOptions.CLUSTERING_ASYNC_ENABLED, true);
+ conf.set(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED, true);
+
+ assertTrue(OptionsResolver.needsAsyncClustering(conf));
+ assertTrue(OptionsResolver.needsScheduleClustering(conf));
+
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+
+ assertFalse(OptionsResolver.needsAsyncClustering(conf));
+ assertFalse(OptionsResolver.needsScheduleClustering(conf));
+ }
}
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
index a4e869428c24..69087122a2e8 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
@@ -359,6 +359,35 @@ public class TestWriteCopyOnWrite extends TestWriteBase {
.end();
}
+ @Test
+ public void testInsertWithTableServiceDisabled() throws Exception {
+ // reset the config option
+ conf.set(FlinkOptions.OPERATION, "insert");
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+ conf.set(FlinkOptions.CLUSTERING_SCHEDULE_ENABLED, true);
+ conf.set(FlinkOptions.CLUSTERING_ASYNC_ENABLED, true);
+ conf.set(FlinkOptions.CLUSTERING_DELTA_COMMITS, 1);
+
+ preparePipeline(conf)
+ .consume(TestData.DATA_SET_INSERT_SAME_KEY)
+ .checkpoint(1)
+ .handleEvents(1)
+ .checkpointComplete(1)
+ .checkWrittenData(EXPECTED4, 1)
+ // insert duplicates again
+ .consume(TestData.DATA_SET_INSERT_SAME_KEY)
+ .checkpoint(2)
+ .handleEvents(1)
+ .checkpointComplete(2)
+ .checkWrittenDataCOW(EXPECTED5)
+ .end();
+ HoodieFlinkWriteClient writeClient = FlinkWriteClients.createWriteClient(conf);
+ long completedReplaceCommit = writeClient.getHoodieTable().getActiveTimeline().getCompletedReplaceTimeline().getInstants().stream().count();
+ long pendingReplaceCommit = writeClient.getHoodieTable().getActiveTimeline().filterPendingReplaceTimeline().getInstants().stream().count();
+ assertEquals(0, completedReplaceCommit);
+ assertEquals(0, pendingReplaceCommit);
+ }
+
@Test
public void testUpsert() throws Exception {
// open the function and ingest data
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
index 7515f8904327..3f24c2763faf 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
@@ -22,6 +22,7 @@ import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.PartialUpdateAvroPayload;
import org.apache.hudi.common.model.WriteConcurrencyMode;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
@@ -42,7 +43,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMPACTION_ACTION;
import static org.apache.hudi.utils.TestData.insertRow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/**
@@ -56,6 +59,34 @@ public class TestWriteMergeOnReadWithCompact extends TestWriteCopyOnWrite {
conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1);
}
+ @Test
+ public void testUpsertWithTableServiceDisabled() throws Exception {
+ // reset the config option
+ conf.set(FlinkOptions.TABLE_SERVICES_ENABLED, false);
+ conf.set(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, true);
+ conf.set(FlinkOptions.COMPACTION_ASYNC_ENABLED, true);
+ conf.set(FlinkOptions.COMPACTION_DELTA_COMMITS, 1);
+
+ preparePipeline(conf)
+ .consume(TestData.DATA_SET_INSERT)
+ .assertEmptyDataFiles()
+ .checkpoint(1)
+ .handleEvents(1)
+ .checkpointComplete(1)
+ .consume(TestData.DATA_SET_INSERT)
+ .checkpoint(2)
+ .handleEvents(1)
+ .checkpointComplete(2)
+ .end();
+ HoodieFlinkWriteClient writeClient = FlinkWriteClients.createWriteClient(conf);
+ long completedCompaction = writeClient.getHoodieTable().getActiveTimeline().getInstants().stream()
+ .filter(s -> s.getAction().equals(COMPACTION_ACTION))
+ .filter(HoodieInstant::isCompleted).count();
+ long pendingCompaction = writeClient.getHoodieTable().getActiveTimeline().filterPendingCompactionTimeline().getInstants().stream().count();
+ assertEquals(0, completedCompaction);
+ assertEquals(0, pendingCompaction);
+ }
+
@Test
public void testPartialFailover() {
// partial failover is only valid for append mode.
(hudi) 08/12: fix: RLI bootstrap fails due to NPE with cleaner table service (#18836)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4d06094008a7a4d347e80aa88b3f8120850d3407
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Tue May 26 11:27:06 2026 +0530
fix: RLI bootstrap fails due to NPE with cleaner table service (#18836)
Co-authored-by: Lokesh Jain <lj...@192.168.1.148>
(cherry picked from commit 652d95253648f1d7b5437e803a405cfa9d1f6f86)
---
.../metadata/HoodieBackedTableMetadataWriter.java | 24 +++++++++++-
.../TestHoodieBackedTableMetadataWriter.java | 45 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 5d09762909be..7035c40c9b42 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -58,6 +58,7 @@ import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaCache;
import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
@@ -934,6 +935,27 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
.sum();
}
+ /**
+ * Resolves the data schema (with metadata fields added) for use during record index bootstrap.
+ * When the write config does not carry a schema (e.g. table-service operations such as clean),
+ * falls back to resolving the schema from the table's commit history / data files.
+ */
+ static HoodieSchema resolveDataSchemaForRLIBootstrap(HoodieTableMetaClient metaClient, HoodieWriteConfig dataWriteConfig) {
+ String writeSchemaStr = dataWriteConfig.getWriteSchema();
+ HoodieSchema rawSchema;
+ if (writeSchemaStr != null) {
+ rawSchema = HoodieSchema.parse(writeSchemaStr);
+ } else {
+ try {
+ rawSchema = new TableSchemaResolver(metaClient).getTableSchema(false);
+ } catch (Exception e) {
+ throw new HoodieException(
+ String.format("Could not resolve schema for table %s for record index bootstrap", metaClient.getBasePath()), e);
+ }
+ }
+ return HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(rawSchema, dataWriteConfig.allowOperationMetadataField()));
+ }
+
/**
* Fetch record locations from FileSlice snapshot.
*
@@ -971,7 +993,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
final FileSlice fileSlice = partitionAndFileSlice.getValue();
final String fileId = fileSlice.getFileId();
HoodieReaderContext<T> readerContext = readerContextFactory.getContext();
- HoodieSchema dataSchema = HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(HoodieSchema.parse(dataWriteConfig.getWriteSchema()), dataWriteConfig.allowOperationMetadataField()));
+ HoodieSchema dataSchema = resolveDataSchemaForRLIBootstrap(metaClient, dataWriteConfig);
HoodieSchema requestedSchema = metaClient.getTableConfig().populateMetaFields() ? getRecordKeySchema()
: HoodieSchemaUtils.projectSchema(dataSchema, Arrays.asList(metaClient.getTableConfig().getRecordKeyFields().orElse(new String[0])));
Option<InternalSchema> internalSchemaOption = SerDeHelper.fromJson(dataWriteConfig.getInternalSchema());
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java
index 4065f113d518..7fcc2ce9a5d8 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java
@@ -25,7 +25,9 @@ import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -54,6 +56,7 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -64,6 +67,7 @@ import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -377,6 +381,47 @@ class TestHoodieBackedTableMetadataWriter {
assertDoesNotThrow(() -> validateRollbackMethod.invoke(writer, instantToRollback));
}
+ // ---- resolveDataSchemaForRLIBootstrap tests ----
+
+ private static final String SIMPLE_SCHEMA_JSON =
+ "{\"type\":\"record\",\"name\":\"Test\",\"namespace\":\"test\","
+ + "\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}";
+
+ @Test
+ void resolveDataSchemaForRLIBootstrap_usesConfigSchemaWhenPresent() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class);
+ when(writeConfig.getWriteSchema()).thenReturn(SIMPLE_SCHEMA_JSON);
+ when(writeConfig.allowOperationMetadataField()).thenReturn(false);
+
+ HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig);
+
+ assertNotNull(result);
+ // metadata fields (_hoodie_*) should have been prepended
+ assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_")));
+ }
+
+ @Test
+ void resolveDataSchemaForRLIBootstrap_fallsBackToTableSchemaResolverWhenNull() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class);
+ when(writeConfig.getWriteSchema()).thenReturn(null);
+ when(writeConfig.allowOperationMetadataField()).thenReturn(false);
+
+ HoodieSchema tableSchema = HoodieSchema.parse(SIMPLE_SCHEMA_JSON);
+ try (org.mockito.MockedConstruction<TableSchemaResolver> mockedResolver =
+ mockConstruction(TableSchemaResolver.class,
+ (resolver, ctx) -> when(resolver.getTableSchema(false)).thenReturn(tableSchema))) {
+
+ HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig);
+
+ assertNotNull(result);
+ assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_")));
+ // exactly one TableSchemaResolver was constructed (with metaClient)
+ assertEquals(1, mockedResolver.constructed().size());
+ }
+ }
+
@SuppressWarnings("deprecation")
private HoodieActiveTimeline createMockTimeline(List<HoodieInstant> instants) {
ActiveTimelineV2 timeline = new ActiveTimelineV2();
(hudi) 12/12: feat(trino): [RFC-105] Trino Hudi Connector — Shim/Bundle Refactor (#18782)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit fbb5c1f1d90e7411a3d51e1e9355bbd79ed686fa
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Tue May 26 20:12:18 2026 -0700
feat(trino): [RFC-105] Trino Hudi Connector — Shim/Bundle Refactor (#18782)
(cherry picked from commit e5ae1115458c99696399b7df62d48f6c00764426)
---
rfc/rfc-105/rfc-105.md | 225 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 225 insertions(+)
diff --git a/rfc/rfc-105/rfc-105.md b/rfc/rfc-105/rfc-105.md
new file mode 100644
index 000000000000..51c7aef2fb90
--- /dev/null
+++ b/rfc/rfc-105/rfc-105.md
@@ -0,0 +1,225 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# RFC-105: Trino Hudi Connector — Shim/Bundle Refactor
+
+## Proposers
+
+- @yihua
+- @voonhous
+
+## Approvers
+
+- @codope
+- @vinothchandar
+
+## Status
+
+Issue: [HUDI-18780](https://github.com/apache/hudi/issues/18780)
+
+> Please keep the status updated in `rfc/README.md`.
+
+## Motivation
+
+The Trino-Hudi connector currently lives in `trinodb/trino` at `plugin/trino-hudi`. Maintaining and evolving the connector through the Trino-OSS-only path has stalled in practice, and the cost falls on Hudi users:
+
+- **Hudi-side improvement PRs to the Trino Hudi connector are not landing.** Four stacked PRs targeting the Trino Hudi connector were closed by Trino's stale-bot for lack of review:
+ - [trinodb/trino#28518](https://github.com/trinodb/trino/pull/28518)
+ - [trinodb/trino#28533](https://github.com/trinodb/trino/pull/28533)
+ - [trinodb/trino#28644](https://github.com/trinodb/trino/pull/28644)
+ - [trinodb/trino#28645](https://github.com/trinodb/trino/pull/28645)
+- **Significant Hudi-side work for the Trino connector is ready but cannot land** through the current path: metadata-table-driven partition listing, eight `HudiIndexSupport` strategies (column stats, partition stats, record-level, secondary, expression, bloom, bucket, partition bloom), MOR snapshot-isolation fixes (worker-side use of the latest commit time from the table handle), and file-system caching integration.
+- **The current arrangement does not scale.** Connector evolution must go through Trino-side review for every change, while the expertise and the source-of-truth for Hudi internals live in this project. Hudi releases cannot directly deliver improvements to Hudi users querying via Trino.
+
+Following alignment between the Hudi and Trino communities, the agreed direction is to split the connector into a thin Trino-side shim plus a Hudi-published artifact carrying the connector logic. This lets the Hudi project ship Trino-Hudi improvements with each Hudi release, while Trino picks them up via a one-line dependency-version bump.
+
+The single requirement carried over from the Trino side is that a comprehensive test suite for the connector continues to be maintained on the Trino side. This RFC documents the agreed approach and the implementation plan.
+
+## Abstract
+
+We split the Trino-Hudi connector into two Maven artifacts:
+
+1. **`io.trino:trino-hudi`** stays in Trino OSS (`plugin/trino-hudi`) as a thin shim — a `HudiPlugin` class that registers the `io.trino.spi.Plugin` SPI entry point — plus the test harness (smoke tests, query runners, MinIO-backed integration tests). This module mostly does not change once landed.
+2. **`org.apache.hudi:hudi-trino`** is a new Hudi-published Maven artifact (regular, non-shaded JAR) containing the actual connector logic at `io.trino.plugin.hudi.*` — `HudiConnectorFactory`, `HudiConnector`, `HudiMetadata`, `HudiSplitManager`, `HudiPageSourceProvider`, all index-support strategies, the `HoodieStorage`/`HoodieIOFactory` bridges to Trino's filesystem, etc. The artifact is built against the latest Trino release's SPI; it declares `hudi-common`, `hudi-io`, etc. as transiti [...]
+
+The first publication ships in **Hudi 1.3.0**. The Trino-side shim PR pins `org.apache.hudi:hudi-trino:1.3.0`. Going forward, all Trino-Hudi connector evolution happens in Hudi OSS; Trino picks up changes by bumping the dependency version. To support this integration model, **Hudi will increase its release cadence**.
+
+## Background
+
+### State of the Trino-side connector today
+
+`plugin/trino-hudi` in `trinodb/trino` is the baseline: it implements the standard Trino SPI (`Plugin`, `ConnectorFactory`, `Connector`, `ConnectorMetadata`, `ConnectorSplitManager`, `ConnectorPageSourceProvider`, etc.), depends on `hudi-common` and `hudi-io`, and uses Hudi's `HoodieStorage` abstraction (RFC-74) over Trino's `TrinoFileSystem`. No direct Hadoop imports.
+
+### State of the Hudi-side `hudi-trino-plugin` work
+
+A more advanced version of the connector exists in Hudi-side branches under `hudi-trino-plugin/` (same `io.trino.plugin.hudi.*` package, built against a recent Trino release). On top of the Trino-OSS baseline it adds:
+
+- Eight `HudiIndexSupport` strategies (column stats, partition stats, record-level, secondary, expression, bloom, bucket, partition bloom) for file- and partition-level pruning via metadata tables.
+- Metadata-table-driven partition discovery (async, resumable).
+- MOR record-level merging via `HoodieFileGroupReader` (`HudiTrinoReaderContext`).
+- Lazy commit-time on `HudiTableHandle` for snapshot-isolated MOR reads across workers.
+- Background, weighted split generation; size-based split weighting; multi-reader routing (`HudiPageSource` for MOR, `HudiBaseFileOnlyPageSource` for COW/RO).
+- File-system cache integration.
+- HoodieStorage / HoodieIOFactory bridges over `TrinoFileSystem` (`HudiTrinoStorage`, `HudiTrinoInlineStorage`, `HudiTrinoIOFactory`).
+
+This is the body of code that will move into the `hudi-trino` Maven module on the Hudi side.
+
+### Why a "shim + Hudi-published artifact" pattern
+
+This pattern decouples Trino-Hudi connector evolution from the Trino-side release cycle:
+
+- The Hudi project can publish Trino-Hudi improvements with each Hudi release, without waiting for Trino-side reviews of every change.
+- The Trino-side surface shrinks to a stable plugin-registration shim, so Trino-side review burden is minimal — typically a one-line version bump per Hudi release.
+- All Hudi-Trino integration code (`io.trino.plugin.hudi.*`) is co-located with the Hudi core libraries it depends on. Changes that cross the Hudi-internal / connector boundary can land atomically.
+- The artifact is **purpose-built for Trino** and implements Trino's SPI directly, so no intermediate adapter layer is needed between the published artifact and the Trino plugin.
+
+Trino's `trino-spi` is governed by `revapi-maven-plugin` (see `core/trino-spi/pom.xml`) which enforces backward compatibility on the SPI surface. This is what makes a single `hudi-trino` artifact targeting the latest Trino release viable across multiple subsequent Trino releases.
+
+Trino loads each plugin in an isolated `URLClassLoader`. Transitive dependencies of `hudi-trino` (Avro, Parquet, etc.) are isolated to the plugin's classloader and cannot conflict with other plugins.
+
+## Implementation
+
+### Architecture
+
+```
+trinodb/trino : plugin/trino-hudi (packaging = trino-plugin)
+ HudiPlugin.java ← thin shim: trivial Plugin SPI registration
+ META-INF/services/io.trino.spi.Plugin
+ src/test/java/... ← full Trino-side test suite
+ pom.xml ← depends on org.apache.hudi:hudi-trino:1.3.0
+ │
+ │ Maven Central
+ ▼
+apache/hudi : hudi-trino-plugin/ (Maven profile -Phudi-trino,
+ excluded from default reactor,
+ JDK 25 required)
+ io.trino.plugin.hudi.* ← all connector logic:
+ HudiConnectorFactory, HudiConnector, HudiMetadata,
+ HudiSplitManager, HudiPageSourceProvider,
+ cache/, file/, io/, partition/,
+ query/ (incl. 8 index-support strategies),
+ reader/, split/, stats/, storage/, util/
+ src/test/java/... ← full duplicated + expanded suite
+ Published as: org.apache.hudi:hudi-trino:1.3.0
+```
+
+### What lives where
+
+#### Trino-side `plugin/trino-hudi` (the shim)
+
+| File | Purpose |
+|---|---|
+| `src/main/java/io/trino/plugin/hudi/HudiPlugin.java` | Implements `io.trino.spi.Plugin`. Single method returning `new HudiConnectorFactory()` (from the `hudi-trino` artifact). ~10 lines. |
+| `src/main/resources/META-INF/services/io.trino.spi.Plugin` | Service-loader pointer to `io.trino.plugin.hudi.HudiPlugin`. |
+| `pom.xml` | `<packaging>trino-plugin</packaging>`; pins `org.apache.hudi:hudi-trino:<version>`; SPI deps as `provided`. |
+| `src/test/java/...` | All current Trino-side tests stay: `HudiQueryRunner`, `TestHudiSmokeTest`, `TestHudiMinioConnectorSmokeTest`, `TestHudiConnectorTest`, `TestHudiSharedMetastore`, `TestHudiSystemTables`, `TestHudiPlugin`, `TestHudiConfig`, plus data initializers. Required by the Trino-side test-coverage commitment. |
+
+#### Hudi-side `hudi-trino-plugin/` (the engine)
+
+Everything else from the current `hudi-trino-plugin/` work, organized exactly as it is today:
+
+| Subpackage | Responsibility |
+|---|---|
+| `io.trino.plugin.hudi` | `HudiConnectorFactory`, `HudiConnector`, `HudiMetadata`, `HudiSplitManager`, `HudiPageSourceProvider`, `HudiSplit`, `HudiTableHandle`, `HudiModule`, `HudiConfig`, `HudiSessionProperties`, `HudiTableProperties`, `HudiTransactionManager`, `HudiMetadataFactory`. |
+| `.cache` | `HudiCacheKeyProvider` for file-system cache integration. |
+| `.file` | `HudiBaseFile`, `HudiLogFile`, file metadata abstractions. |
+| `.io` | `HudiTrinoIOFactory` (extends `HoodieIOFactory`), `HudiTrinoFileReaderFactory`, `TrinoSeekableDataInputStream`. |
+| `.partition` | `HudiPartitionInfo`, `HiveHudiPartitionInfo`, `HudiPartitionInfoLoader` (async resumable task). |
+| `.query` | `HudiDirectoryLister`, `HudiReadOptimizedDirectoryLister`, `HudiSnapshotDirectoryLister`; `query.index` package with 8 `HudiIndexSupport` strategies. |
+| `.reader` | `HudiTrinoReaderContext extends HoodieReaderContext<IndexedRecord>` for MOR record merging. |
+| `.split` | `HudiSplitFactory`, `HudiBackgroundSplitLoader`, `HudiSplitSource`, `HudiSplitWeightProvider`, `SizeBasedSplitWeightProvider`. |
+| `.stats` | `HudiTableStatistics`, `TableStatisticsReader`. |
+| `.storage` | `HudiTrinoStorage` (extends `HoodieStorage`), `HudiTrinoInlineStorage`, `TrinoStorageConfiguration`. |
+| `.util` | Serialization helpers, column synthesis, tuple-domain conversion, table-type utilities. |
+
+### API boundary
+
+The boundary between the shim and the published artifact is **Trino's SPI itself** — no intermediate API layer is introduced.
+
+- **Shim → artifact:** `HudiPlugin.getConnectorFactories()` returns `new HudiConnectorFactory()` defined in the artifact. Trino's runtime then calls `factory.create(catalogName, config, context)`. The `ConnectorContext` argument carries everything the artifact needs — `TypeManager`, `NodeManager`, `MetadataProvider`, `PageSorter`, `PageIndexerFactory`, `OpenTelemetry`, `Tracer`, `CatalogHandle` — without the artifact importing implementation classes.
+- **Artifact → Trino:** the artifact's `HudiConnector` exposes the standard SPI providers (`ConnectorMetadata`, `ConnectorSplitManager`, `ConnectorPageSourceProvider`, etc.). Trino calls these. Classloader context is handled by the standard `ClassLoaderSafe*` wrappers (`io.trino.plugin.base.classloader.*`) — already used today.
+
+### Maven dependencies for `hudi-trino`
+
+- **`compile`:** Hudi libs (`hudi-common`, `hudi-io`, `hudi-hive-sync`, `hudi-sync-common`) and Trino libs (`trino-filesystem`, `trino-hive`, `trino-metastore`, `trino-parquet`, `trino-cache`), Guice, Airlift, Caffeine.
+- **`provided`:** `trino-spi`, `slice`, Jackson, OpenTelemetry API, JOL (supplied by Trino at runtime).
+- **`runtime`:** log-manager, Dropwizard metrics, OpenTelemetry SDK, `trino-hive-formats`.
+- **`test`:** Trino testing libs (`trino-testing`, `trino-main`, `trino-testing-containers`, `trino-hdfs`), AssertJ, JUnit 5, Hudi test JARs.
+
+**Version alignment policy.** Trino versions are authoritative for shared libraries (Avro, Parquet, Jackson, Airlift). The `hudi-trino` POM pins these via `<dependencyManagement>` to whatever the targeted Trino release uses. If Hudi internals need a newer version, the fix is on the Hudi side or via a Trino-version bump — never by shipping divergent classpath versions.
+
+### Build target on Hudi side
+
+Trino requires Java 25, while the rest of Hudi targets a lower Java floor. `hudi-trino-plugin` therefore lives behind a Maven profile (`-Phudi-trino`) and is **excluded from the default `mvn install` reactor**:
+
+```xml
+<profile>
+ <id>hudi-trino</id>
+ <modules>
+ <module>hudi-trino-plugin</module>
+ </modules>
+</profile>
+```
+
+Default build (`mvn install`) skips it; Trino-targeted build (`mvn install -Phudi-trino`) requires JDK 25.
+
+### CI
+
+Two new GitHub Actions on the Hudi side, required for any change touching `hudi-trino-plugin/**`:
+
+1. **`hudi-trino-ci.yml`** — runs the full test suite via `mvn verify -Phudi-trino` on JDK 25. Catches regressions before they ship in a Hudi release.
+2. **`hudi-trino-compat.yml`** — nightly: pulls latest `trinodb/trino` master and latest `apache/hudi` master, builds Trino's relevant modules, then compiles `hudi-trino-plugin` against them and runs the `hudi-trino-plugin` test suite. Catches both SPI drift and behavioral incompatibilities before the next Trino release.
+
+On the Trino side, existing CI continues to build and test `plugin/trino-hudi`, exercising the published `hudi-trino` artifact end-to-end on every Trino PR.
+
+### Test strategy
+
+**Full test duplication.** The Trino-side smoke tests (`TestHudiSmokeTest`, `TestHudiMinioConnectorSmokeTest`, `TestHudiConnectorTest`, etc.) are mirrored on the Hudi side and additionally extended.
+
+- **Trino side runs them** on every Trino PR — fulfilling the Trino-side test-coverage commitment.
+- **Hudi side runs them** on every Hudi PR touching `hudi-trino-plugin` — so Hudi contributors catch regressions before they ship in a Hudi release. The Hudi-side suite is also **expanded** with more granular unit tests covering split generation edge cases, all eight index-support strategies, the MOR record-merging path, lazy-commit-time snapshot isolation, and the cache-key provider.
+
+This duplication has a known cost — two places to update when adding tests — but is the right trade-off given:
+- The Trino-side suite must remain comprehensive as agreed with the Trino community.
+- Hudi-side contributors need fast feedback without waiting for a Trino-side PR cycle.
+
+### Risks & caveats
+
+- **Trino SPI drift.** A future Trino SPI change could break the pre-built `hudi-trino` artifact at runtime. Mitigation: the nightly compat CI compiles and runs the `hudi-trino-plugin` test suite against Trino master, flagging both compile-time and behavioral incompatibilities before a Trino release ships.
+- **Avro / Parquet / Jackson version skew.** Resolved by policy: Trino's versions are authoritative, pinned via `<dependencyManagement>` in the `hudi-trino` POM. Hudi-side fixes or Trino-version bumps adjust to it.
+- **Test-infrastructure coupling.** `hudi-trino-plugin`'s test scope depends on `trino-testing`, `trino-main`, etc., coupling the Hudi build to Trino artifacts on Maven Central. Acceptable cost.
+- **Release coordination.** A critical fix in `hudi-trino` ships only via a Hudi release. Mitigation: keep the Trino-side shim trivial so virtually all fixes can land in `hudi-trino`, and increase Hudi release cadence.
+- **License / ASF process.** Cross-project releases between two ASF projects; covered by standard PMC announcements at first release.
+
+## Rollout/Adoption Plan
+
+**Step 1 — Hudi 1.3.0 publishes `hudi-trino`.** Land the `hudi-trino-plugin` work in `apache/hudi` master behind the `-Phudi-trino` profile, land the two CI workflows, then publish `org.apache.hudi:hudi-trino:1.3.0` to Maven Central as part of the 1.3.0 release. Hudi commits to a more frequent release cadence going forward: to start, a major release roughly every month, with more frequent minor releases to stabilize this module. A `hudi-trino` artifact is cut whenever there are enough im [...]
+
+**Step 2 — Trino-side shim PR.** A small PR against `trinodb/trino` that replaces the contents of `plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/` with a single `HudiPlugin.java`, keeps `META-INF/services/io.trino.spi.Plugin` and all current tests, and adds `org.apache.hudi:hudi-trino:1.3.0` as a `compile` dependency. The PR is small by design — deletes connector code, points at the published artifact — so Trino-maintainer review burden is minimal.
+
+**Step 3 — Steady state.** Trino-Hudi feature work and bug fixes happen on the Hudi side. Each Hudi release publishes a new `hudi-trino` artifact. Trino picks up changes by bumping the pinned version — a one-line PR per release. Because Trino pins `hudi-trino` as a compile-time dependency, the `hudi-trino` ↔ Trino release mapping is **one-to-one**: each `hudi-trino` version is the version that ships embedded in a specific Trino release. This one-to-one mapping makes feature support and b [...]
+
+**Impact on existing users.** No behavioral change: same `HudiPlugin` registration, same catalog config. The first Trino release picking up `hudi-trino:1.3.0` gains the features that the previously-stalled PRs covered (metadata-table partition listing, index support, MOR snapshot-isolation correctness, file-system caching, advanced split generation). No migration tools needed.
+
+## Test Plan
+
+The RFC is validated when:
+
+- [ ] `hudi-trino-plugin` builds and its full test suite passes on the Hudi side via `mvn verify -Phudi-trino` (JDK 25), covering smoke tests, MinIO/Alluxio caching tests, MOR/COW tests, page source tests, split-factory tests, index support tests, system-table tests, and plugin/config tests.
+- [ ] The `hudi-trino-compat.yml` workflow compiles `hudi-trino-plugin` against `trinodb/trino` master successfully.
+- [ ] `org.apache.hudi:hudi-trino:1.3.0` is published to Maven Central.
+- [ ] The Trino-side shim PR is green: Trino's CI for `plugin/trino-hudi` passes against the published `1.3.0` artifact, including MinIO/S3 integration and plugin-loading tests.
+- [ ] At least one subsequent Hudi-side patch release exercises the "bump version in Trino" steady-state flow end-to-end.
(hudi) 01/12: feat(spark): add show_inflight_commits and cleanup_stale_inflight_com… (#18709)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8f74fdaf2f5245210e620bea62a95a8fc1dc5343
Author: Mahsood Ebrahim <ma...@gmail.com>
AuthorDate: Tue May 19 14:44:27 2026 +0800
feat(spark): add show_inflight_commits and cleanup_stale_inflight_com… (#18709)
* feat(spark): add show_inflight_commits and cleanup_stale_inflight_commits stored procedures
Two new CALL procedures so operators can inspect and remediate stale
inflight commits via SQL instead of using hudi-cli.
show_inflight_commits(table, min_age_minutes?) lists REQUESTED+INFLIGHT
instants from the active timeline.
cleanup_stale_inflight_commits(table, allowed_inflight_interval_minutes?,
include_ingestion_commits?, dry_run?) rolls back stale write-timeline
inflights older than the threshold (default 180 min). COMPACTION,
LOG_COMPACTION, and CLUSTERING route to their dedicated
table.rollbackInflight* methods (HoodieSparkTable is lazy-init on first
such instant); other write actions go through client.rollback().
include_ingestion_commits and dry_run both default to false; dry_run
emits rollback_status=NULL and skips write-client construction.
A single-method utility HoodieTimelineCleanupUtil
(inflightWriteCommitsOlderThan) is added in hudi-spark-common.
Tested with 4 show + 9 cleanup unit tests covering empty/threshold/
ingestion-gating/dry_run paths plus COMPACTION, CLUSTERING, partitioned
COW, and MOR delta_commit. checkstyle + scalastyle clean.
* address review comments on show_inflight_commits / cleanup_stale_inflight_commits
- Rename HoodieTimelineCleanupUtil.inflightWriteCommitsOlderThan param
`mins` -> `ageMinutes` for clarity.
- Replace Duration.ofMinutes(...).getSeconds() * 1000 with .toMillis()
in HoodieTimelineCleanupUtil and ShowInflightCommitsProcedure.
- Add inflight-state recheck in CleanupStaleInflightCommitsProcedure
before client.rollback(): reloads the active timeline and skips the
rollback if the instant is no longer INFLIGHT/REQUESTED, preventing
destructive rollback of a commit that completed concurrently after
detection.
---------
Co-authored-by: mahsoode <ma...@uber.com>
(cherry picked from commit d94b2e2c3162fefd26a7c86301228f6c2c74ceef)
---
.../org/apache/hudi/HoodieTimelineCleanupUtil.java | 59 +++
.../CleanupStaleInflightCommitsProcedure.scala | 224 +++++++++++
.../hudi/command/procedures/HoodieProcedures.scala | 2 +
.../procedures/ShowInflightCommitsProcedure.scala | 113 ++++++
.../TestCleanupStaleInflightCommitsProcedure.scala | 413 +++++++++++++++++++++
.../TestShowInflightCommitsProcedure.scala | 151 ++++++++
6 files changed, 962 insertions(+)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java
new file mode 100644
index 000000000000..f8583d5d5d2a
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi;
+
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.Date;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class HoodieTimelineCleanupUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(HoodieTimelineCleanupUtil.class);
+
+ public static List<HoodieInstant> inflightWriteCommitsOlderThan(HoodieTableMetaClient metaClient, long ageMinutes, boolean includeIngestionCommits) {
+ long goBackMs = Duration.ofMinutes(ageMinutes).toMillis();
+ String oldestAllowedTimestamp = HoodieInstantTimeGenerator.formatDate(new Date(System.currentTimeMillis() - goBackMs));
+
+ Stream<HoodieInstant> inflightInstants = metaClient
+ .reloadActiveTimeline()
+ .getWriteTimeline()
+ .filterInflightsAndRequested()
+ .findInstantsBefore(oldestAllowedTimestamp)
+ .getInstants().stream();
+
+ if (!includeIngestionCommits) {
+ Predicate<HoodieInstant> ingestionCommitsFilter =
+ (x) -> x.getAction().equals(HoodieTimeline.COMMIT_ACTION) || x.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION);
+ inflightInstants = inflightInstants.filter(ingestionCommitsFilter.negate());
+ }
+ List<HoodieInstant> inflightInstantsList = inflightInstants.collect(Collectors.toList());
+ LOG.info("Inflight commits older than {} minutes: {}", ageMinutes, inflightInstantsList);
+ return inflightInstantsList;
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala
new file mode 100644
index 000000000000..66e31302e45b
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.command.procedures
+
+import org.apache.hudi.{HoodieCLIUtils, HoodieTimelineCleanupUtil}
+import org.apache.hudi.client.SparkRDDWriteClient
+import org.apache.hudi.common.HoodiePendingRollbackInfo
+import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.table.timeline.HoodieTimeline
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.HoodieWriteConfig
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+import org.apache.hudi.table.HoodieSparkTable
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
+
+import java.util.function.Supplier
+
+import scala.collection.JavaConverters._
+
+/**
+ * Spark SQL stored procedure to roll back stale inflight write commits older than a configurable
+ * age threshold.
+ *
+ * SAFETY WARNING: Unlike rollback_to_instant_time which targets a single named instant, this
+ * procedure rolls back a SET of instants matched by age. A misconfigured threshold can erase
+ * recent in-progress writes when include_ingestion_commits=true. Operators should call
+ * show_inflight_commits first to preview the pending instants, and use dry_run => true (see
+ * below) when in doubt about which instants will be processed.
+ *
+ * This procedure targets the write timeline (COMMIT, DELTA_COMMIT, COMPACTION, LOG_COMPACTION,
+ * REPLACE_COMMIT, CLUSTERING actions). Stale rollback, clean, or restore inflights visible in
+ * show_inflight_commits are NOT covered by this procedure.
+ *
+ * Compaction, log-compaction, and clustering inflights are handled via the targeted table
+ * methods (table.rollbackInflightCompaction / rollbackInflightLogCompaction /
+ * rollbackInflightClustering) — the supporting state (HoodieSparkTable, table-service client,
+ * pending-rollback lookup) is constructed lazily on the first such instant and reused.
+ *
+ * Parameters:
+ * - table: Required. Catalog name of the Hudi table.
+ * - allowed_inflight_interval_minutes: Optional (default 180). Instants older than this many
+ * minutes are considered stale and eligible for rollback.
+ * - include_ingestion_commits: Optional (default false). DANGEROUS when true: enabling
+ * this allows the procedure to roll back COMMIT_ACTION and
+ * DELTA_COMMIT_ACTION inflights, which means it can drop
+ * in-progress ingestion data. The default false is the
+ * safe choice; true is for operators recovering from a
+ * known-stuck ingestion job.
+ * - dry_run: Optional (default false). When true, the matched-instant
+ * set is resolved exactly as in normal mode but no rollback
+ * calls are issued. Each returned row carries
+ * rollback_status = NULL meaning "matched but not acted
+ * upon". Re-run with dry_run => false to act.
+ *
+ * Output columns (one row per processed instant):
+ * - instant_time: The instant's requested timestamp.
+ * - action: The action type.
+ * - rollback_status: true if the rollback succeeded, false if the rollback failed,
+ * NULL if dry_run was true (matched but not actioned).
+ *
+ * Example usage:
+ * {{{
+ * -- Clean stale table-service inflights (default 180-min threshold)
+ * CALL cleanup_stale_inflight_commits(table => 'my_table');
+ *
+ * -- Preview what would be processed without acting
+ * CALL cleanup_stale_inflight_commits(table => 'my_table', dry_run => true);
+ *
+ * -- Clean stale inflights older than 1 hour, including ingestion commits (DANGEROUS)
+ * CALL cleanup_stale_inflight_commits(
+ * table => 'my_table',
+ * allowed_inflight_interval_minutes => 60,
+ * include_ingestion_commits => true
+ * );
+ * }}}
+ *
+ * For inflight commit types not covered by this procedure (clean, restore, rollback inflights),
+ * use hudi-cli's `repair rollback` command.
+ */
+class CleanupStaleInflightCommitsProcedure extends BaseProcedure with ProcedureBuilder with Logging {
+
+ private val PARAMETERS = Array[ProcedureParameter](
+ ProcedureParameter.required(0, "table", DataTypes.StringType),
+ ProcedureParameter.optional(1, "allowed_inflight_interval_minutes", DataTypes.IntegerType, 180),
+ ProcedureParameter.optional(2, "include_ingestion_commits", DataTypes.BooleanType, false),
+ ProcedureParameter.optional(3, "dry_run", DataTypes.BooleanType, false)
+ )
+
+ private val OUTPUT_TYPE = new StructType(Array[StructField](
+ StructField("instant_time", DataTypes.StringType, nullable = true, Metadata.empty),
+ StructField("action", DataTypes.StringType, nullable = true, Metadata.empty),
+ StructField("rollback_status", DataTypes.BooleanType, nullable = true, Metadata.empty)
+ ))
+
+ override def parameters: Array[ProcedureParameter] = PARAMETERS
+
+ override def outputType: StructType = OUTPUT_TYPE
+
+ override def build: Procedure = new CleanupStaleInflightCommitsProcedure()
+
+ override def call(args: ProcedureArgs): Seq[Row] = {
+ super.checkArgs(PARAMETERS, args)
+
+ val tableName = getArgValueOrDefault(args, PARAMETERS(0))
+ val allowedMinutes = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Int]
+ val includeIngestionCommits = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Boolean]
+ val dryRun = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Boolean]
+
+ val basePath = getBasePath(tableName)
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration))
+ .setBasePath(basePath)
+ .build
+
+ val staleInflights = HoodieTimelineCleanupUtil
+ .inflightWriteCommitsOlderThan(metaClient, allowedMinutes.toLong, includeIngestionCommits)
+
+ if (staleInflights.isEmpty) {
+ Seq.empty[Row]
+ } else if (dryRun) {
+ // Dry-run: do not open the table for write — emit preview rows with NULL rollback_status
+ // to mean "matched but not actioned". Re-run with dry_run => false to act.
+ staleInflights.asScala.map { instant =>
+ Row(instant.requestedTime, instant.getAction, null)
+ }.toSeq
+ } else {
+ // Pass ROLLBACK_USING_MARKERS_ENABLE=false via the createHoodieWriteClient confs Map.
+ // Inflight commits may not have marker files, so timeline-based rollback is required.
+ // The user-specified confs win over defaults / table config / session conf — see
+ // HoodieCLIUtils.scala "Priority: defaults < catalog props < table config < sparkSession conf < specified conf".
+ val confs = Map(HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.key() -> "false")
+ var client: SparkRDDWriteClient[_] = null
+ try {
+ client = HoodieCLIUtils.createHoodieWriteClient(sparkSession, basePath, confs,
+ tableName.asInstanceOf[scala.Option[String]])
+
+ // Lazy state for compaction / log-compaction / clustering branches.
+ // For matched sets that contain no such instants, none of these are constructed.
+ lazy val tsClient = client.getTableServiceClient
+ lazy val table = HoodieSparkTable.create(client.getConfig, client.getEngineContext)
+ lazy val getPendingRollbackInstantFunc: java.util.function.Function[String, HOption[HoodiePendingRollbackInfo]] =
+ new java.util.function.Function[String, HOption[HoodiePendingRollbackInfo]] {
+ override def apply(commitToRollback: String): HOption[HoodiePendingRollbackInfo] = {
+ tsClient.getPendingRollbackInfo(table.getMetaClient, commitToRollback, false)
+ }
+ }
+
+ val rows = staleInflights.asScala.map { instant =>
+ val status: java.lang.Boolean = try {
+ val result: Boolean = instant.getAction match {
+ case HoodieTimeline.COMPACTION_ACTION =>
+ table.rollbackInflightCompaction(instant, getPendingRollbackInstantFunc, client.getTransactionManager)
+ true
+ case HoodieTimeline.LOG_COMPACTION_ACTION =>
+ table.rollbackInflightLogCompaction(instant, getPendingRollbackInstantFunc, client.getTransactionManager)
+ true
+ case HoodieTimeline.CLUSTERING_ACTION =>
+ table.rollbackInflightClustering(instant, getPendingRollbackInstantFunc, client.getTransactionManager)
+ true
+ case _ =>
+ // Recheck that the instant is still inflight before calling client.rollback(),
+ // which searches getCommitsTimeline() (completed + pending). Without this guard,
+ // a concurrent writer that completes the commit after detection could have its
+ // now-completed commit rolled back destructively. The table.rollbackInflight*
+ // branches above fail loudly via revertInstantFromInflightToRequested if the
+ // inflight is gone; this branch performs an equivalent safety check.
+ val stillInflight = metaClient.reloadActiveTimeline()
+ .filterInflightsAndRequested()
+ .containsInstant(instant.requestedTime)
+ if (!stillInflight) {
+ logWarning(s"Instant ${instant.requestedTime} is no longer inflight; " +
+ "skipping rollback to avoid rolling back a completed commit")
+ false
+ } else {
+ client.rollback(instant.requestedTime)
+ }
+ }
+ java.lang.Boolean.valueOf(result)
+ } catch {
+ case e: Exception =>
+ logError(s"Failed to rollback inflight instant ${instant.requestedTime}", e)
+ java.lang.Boolean.FALSE
+ }
+ Row(instant.requestedTime, instant.getAction, status)
+ }.toSeq
+
+ // Refresh catalog after all rollbacks, inside try — consistent with RollbackToInstantTimeProcedure.
+ // Not placed in finally to avoid refreshing on client-creation failures.
+ if (tableName.isDefined) {
+ spark.catalog.refreshTable(tableName.get.asInstanceOf[String])
+ }
+ rows
+ } finally {
+ if (client != null) client.close()
+ }
+ }
+ }
+}
+
+object CleanupStaleInflightCommitsProcedure {
+ val NAME = "cleanup_stale_inflight_commits"
+
+ def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] {
+ override def get() = new CleanupStaleInflightCommitsProcedure()
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala
index 027c9f8c2882..fd084ec0bf6e 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedures.scala
@@ -94,6 +94,8 @@ object HoodieProcedures {
,(ShowTablePropertiesProcedure.NAME, ShowTablePropertiesProcedure.builder)
,(HelpProcedure.NAME, HelpProcedure.builder)
,(ArchiveCommitsProcedure.NAME, ArchiveCommitsProcedure.builder)
+ ,(ShowInflightCommitsProcedure.NAME, ShowInflightCommitsProcedure.builder)
+ ,(CleanupStaleInflightCommitsProcedure.NAME, CleanupStaleInflightCommitsProcedure.builder)
,(RunTTLProcedure.NAME, RunTTLProcedure.builder)
,(DropPartitionProcedure.NAME, DropPartitionProcedure.builder)
,(TruncateTableProcedure.NAME, TruncateTableProcedure.builder)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala
new file mode 100644
index 000000000000..3b501bdb25a1
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.command.procedures
+
+import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
+
+import java.time.Duration
+import java.util.Date
+import java.util.function.Supplier
+
+import scala.collection.JavaConverters._
+
+/**
+ * Spark SQL stored procedure to list all pending inflight and requested instants on a Hudi table.
+ *
+ * Unlike cleanup_stale_inflight_commits (which targets the write timeline only), this procedure
+ * queries the full active timeline and therefore includes inflight rollback, clean, restore, and
+ * indexing instants in addition to write-action instants. A stale rollback or clean inflight
+ * visible here cannot be cleaned by cleanup_stale_inflight_commits.
+ *
+ * Parameters:
+ * - table: Required. Catalog name of the Hudi table.
+ * - min_age_minutes: Optional (default 0). When > 0, only instants older than this many minutes
+ * are returned. When 0 (default), all pending instants are returned.
+ *
+ * Output columns (one row per pending instant):
+ * - instant_time: The instant's requested timestamp.
+ * - action: The action type (commit, delta_commit, compaction, replace, rollback, clean, etc.).
+ * - state: The instant's state (REQUESTED or INFLIGHT).
+ *
+ * Example usage:
+ * {{{
+ * -- Show all pending inflights
+ * CALL show_inflight_commits(table => 'my_table');
+ *
+ * -- Show only inflights older than 2 hours
+ * CALL show_inflight_commits(table => 'my_table', min_age_minutes => 120);
+ * }}}
+ */
+class ShowInflightCommitsProcedure extends BaseProcedure with ProcedureBuilder {
+
+ private val PARAMETERS = Array[ProcedureParameter](
+ ProcedureParameter.required(0, "table", DataTypes.StringType),
+ ProcedureParameter.optional(1, "min_age_minutes", DataTypes.IntegerType, 0)
+ )
+
+ private val OUTPUT_TYPE = new StructType(Array[StructField](
+ StructField("instant_time", DataTypes.StringType, nullable = true, Metadata.empty),
+ StructField("action", DataTypes.StringType, nullable = true, Metadata.empty),
+ StructField("state", DataTypes.StringType, nullable = true, Metadata.empty)
+ ))
+
+ override def parameters: Array[ProcedureParameter] = PARAMETERS
+
+ override def outputType: StructType = OUTPUT_TYPE
+
+ override def build: Procedure = new ShowInflightCommitsProcedure()
+
+ override def call(args: ProcedureArgs): Seq[Row] = {
+ super.checkArgs(PARAMETERS, args)
+
+ val tableName = getArgValueOrDefault(args, PARAMETERS(0))
+ val minAgeMinutes = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Int]
+
+ val basePath = getBasePath(tableName)
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(jsc.hadoopConfiguration))
+ .setBasePath(basePath)
+ .build
+
+ val baseTimeline = metaClient.reloadActiveTimeline().filterInflightsAndRequested()
+
+ val timeline = if (minAgeMinutes > 0) {
+ val goBackMs = Duration.ofMinutes(minAgeMinutes).toMillis
+ val cutoff = HoodieInstantTimeGenerator.formatDate(new Date(System.currentTimeMillis() - goBackMs))
+ baseTimeline.findInstantsBefore(cutoff)
+ } else {
+ baseTimeline
+ }
+
+ timeline.getInstants.asScala.map { instant =>
+ Row(instant.requestedTime, instant.getAction, instant.getState.name())
+ }.toSeq
+ }
+}
+
+object ShowInflightCommitsProcedure {
+ val NAME = "show_inflight_commits"
+
+ def builder: Supplier[ProcedureBuilder] = new Supplier[ProcedureBuilder] {
+ override def get() = new ShowInflightCommitsProcedure()
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala
new file mode 100644
index 000000000000..f18a173e29ad
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.procedure
+
+import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+
+import java.text.SimpleDateFormat
+import java.util.Date
+
+import scala.collection.JavaConverters._
+
+class TestCleanupStaleInflightCommitsProcedure extends HoodieSparkProcedureTestBase {
+
+ /**
+ * Creates a table DDL without inserting data. Tests that manipulate inflight instants must NOT
+ * insert data before injecting the inflight, because BaseRollbackActionExecutor
+ * .validateRollbackCommitSequence throws HoodieRollbackException when committed instants exist
+ * after the injected (old) timestamp and no heartbeat exists for the injected instant.
+ * With no prior inserts, commitTimeline.empty() = true and the guard is bypassed.
+ */
+ private def createEmptyTable(tableName: String, tablePath: String): Unit = {
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | ts long
+ | ) using hudi
+ | location '$tablePath'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+ }
+
+ private def createEmptyPartitionedTable(tableName: String, tablePath: String, tableType: String): Unit = {
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | ts long,
+ | part int
+ | ) using hudi
+ | partitioned by (part)
+ | location '$tablePath'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = '$tableType',
+ | preCombineField = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+ }
+
+ test("Test cleanup_stale_inflight_commits returns empty when no stale inflights exist") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ createEmptyTable(tableName, tmp.getCanonicalPath)
+ spark.sql(s"insert into $tableName values(1, 'a1', 1000)")
+ spark.sql(s"insert into $tableName values(2, 'a2', 2000)")
+
+ val result = spark.sql(s"call cleanup_stale_inflight_commits(table => '$tableName')").collect()
+ assertResult(0)(result.length)
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits rolls back stale REPLACE_COMMIT_ACTION inflight") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ // No inserts before injecting — see createEmptyTable docstring
+ createEmptyTable(tableName, tablePath)
+
+ val staleTs = "20200101120000"
+ // Must use REPLACE_COMMIT_ACTION: inflightWriteCommitsOlderThan with
+ // include_ingestion_commits=false (default) filters out COMMIT_ACTION and DELTA_COMMIT_ACTION.
+ // REPLACE_COMMIT_ACTION is included in both getWriteTimeline() and getCommitsTimeline(),
+ // so client.rollback() finds it and returns true.
+ injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs)
+
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1))
+ assertResult(true)(result(0).getBoolean(2))
+
+ // Verify the instant is gone from the active timeline
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val remaining = metaClient.reloadActiveTimeline().filterInflightsAndRequested()
+ .getInstants.asScala
+ assert(!remaining.exists(_.requestedTime == staleTs),
+ s"Stale instant $staleTs should have been removed from the timeline after rollback")
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits respects allowed_inflight_interval_minutes threshold") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ // No inserts before injecting — see createEmptyTable docstring
+ createEmptyTable(tableName, tablePath)
+
+ val staleTs = "20200101120000"
+ val freshTs = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date())
+
+ injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs)
+ injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, freshTs)
+
+ // 60-minute threshold: only the stale instant qualifies; fresh instant is too recent
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 60)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits cleans COMMIT_ACTION with include_ingestion_commits=true") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ // No inserts before injecting — see createEmptyTable docstring
+ createEmptyTable(tableName, tablePath)
+
+ val staleTs = "20200101120000"
+ // COMMIT_ACTION is filtered out with the default include_ingestion_commits=false,
+ // but included when include_ingestion_commits=true.
+ // client.rollback returns true for COMMIT_ACTION since getCommitsTimeline() includes it.
+ injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, staleTs)
+
+ // Default (include_ingestion_commits=false): should not find COMMIT_ACTION inflight
+ val defaultResult = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1)").collect()
+ assertResult(0)(defaultResult.length)
+
+ // With include_ingestion_commits=true: should find and process COMMIT_ACTION inflight
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1, " +
+ s"include_ingestion_commits => true)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ assertResult(HoodieTimeline.COMMIT_ACTION)(result(0).getString(1))
+ assertResult(true)(result(0).getBoolean(2))
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits dry_run lists matched instants without rolling back") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ createEmptyTable(tableName, tablePath)
+
+ val staleTs = "20200101120000"
+ injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs)
+
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1, " +
+ s"dry_run => true)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1))
+ // dry_run: rollback_status is NULL meaning "matched but not actioned"
+ assert(result(0).isNullAt(2),
+ "Expected rollback_status=NULL in dry_run mode, but got non-null value")
+
+ // Verify the instant is STILL on the active timeline (dry_run did not act)
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val remaining = metaClient.reloadActiveTimeline().filterInflightsAndRequested()
+ .getInstants.asScala
+ assert(remaining.exists(_.requestedTime == staleTs),
+ s"dry_run should not have rolled back $staleTs; expected to find it still on the timeline")
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits rolls back stale COMPACTION_ACTION inflight via table.rollbackInflightCompaction") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // Compaction is MOR-only. Use the real schedule + run + delete-commit pattern from
+ // TestRunRollbackInflightTableServiceProcedure to construct a valid compaction inflight.
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long
+ |) using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'mor',
+ | preCombineField = 'ts'
+ | )
+ | partitioned by(ts)
+ | location '$tablePath'
+ """.stripMargin)
+ withSQLConf(
+ "hoodie.parquet.max.file.size" -> "10000",
+ "hoodie.compact.inline" -> "false",
+ "hoodie.compact.schedule.inline" -> "false",
+ // Prevent auto-clean from creating a clean instant after compaction completion,
+ // which would shift getReverseOrderedInstants.findFirst() away from the compaction commit.
+ "hoodie.clean.automatic" -> "false"
+ ) {
+ spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
+ spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000)")
+ spark.sql(s"insert into $tableName values(3, 'a3', 10, 1000)")
+ spark.sql(s"insert into $tableName values(4, 'a4', 10, 1000)")
+ spark.sql(s"update $tableName set price = 11 where id = 1")
+
+ spark.sql(s"call run_compaction(op => 'schedule', table => '$tableName')")
+ spark.sql(s"call run_compaction(op => 'run', table => '$tableName')")
+
+ // Delete the completed compaction commit file so the inflight remains
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val compactionInstant = metaClient.getActiveTimeline.getReverseOrderedInstants.findFirst().get()
+ metaClient.getActiveTimeline.deleteInstantFileIfExists(compactionInstant)
+ val compactionInstantTime = compactionInstant.requestedTime
+
+ // Confirm the compaction inflight is actually present before we call cleanup.
+ // If this assertion fires, the test setup (schedule+run+delete) didn't produce the expected
+ // state and the rest of the test is moot — fail with a clear diagnostic instead of an empty result.
+ val reloadedTimeline = metaClient.reloadActiveTimeline()
+ val compactionInflightPresent = reloadedTimeline.getWriteTimeline.filterInflightsAndRequested.getInstants.asScala
+ .exists(i => i.getAction == HoodieTimeline.COMPACTION_ACTION && i.requestedTime == compactionInstantTime)
+ assert(compactionInflightPresent,
+ s"Setup failure: compaction inflight at $compactionInstantTime not present after deleting completed commit. " +
+ s"Active timeline: ${reloadedTimeline.getInstants.asScala.map(i => s"${i.requestedTime}/${i.getAction}/${i.getState}").mkString(", ")}")
+
+ // Sleep so the second-precision cutoff timestamp is strictly newer than the inflight's timestamp
+ Thread.sleep(2000)
+
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 0)").collect()
+
+ val compactionRow = result.find(r => r.getString(0) == compactionInstantTime)
+ assert(compactionRow.isDefined,
+ s"Expected compaction inflight $compactionInstantTime in result; got ${result.map(r => s"${r.getString(0)}/${r.getString(1)}").mkString(",")}")
+ assertResult(HoodieTimeline.COMPACTION_ACTION)(compactionRow.get.getString(1))
+ assertResult(true)(compactionRow.get.getBoolean(2))
+
+ // Inflight should be removed by table.rollbackInflightCompaction
+ val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator
+ val expectedInflight = instantGenerator.createNewInstant(
+ HoodieInstant.State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, compactionInstantTime)
+ assert(!metaClient.reloadActiveTimeline().getInstants.contains(expectedInflight),
+ s"Compaction inflight $compactionInstantTime should be gone after rollback")
+ }
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits rolls back stale CLUSTERING_ACTION inflight via table.rollbackInflightClustering") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // Use the real schedule + execute + delete-commit pattern from
+ // TestRunRollbackInflightTableServiceProcedure so the clustering inflight is valid.
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long
+ |) using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts'
+ | )
+ | partitioned by(ts)
+ | location '$tablePath'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
+ spark.sql(s"insert into $tableName values(2, 'a2', 10, 1001)")
+ spark.sql(s"insert into $tableName values(3, 'a3', 10, 1002)")
+
+ spark.sql(s"call run_clustering(table => '$tableName', op => 'schedule')")
+ spark.sql(s"call run_clustering(table => '$tableName', op => 'execute')")
+
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val clusteringInstant = metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.get(0)
+ metaClient.getActiveTimeline.deleteInstantFileIfExists(clusteringInstant)
+ val clusteringInstantTime = clusteringInstant.requestedTime
+
+ Thread.sleep(2000)
+
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 0)").collect()
+
+ val clusteringRow = result.find(r => r.getString(0) == clusteringInstantTime)
+ assert(clusteringRow.isDefined,
+ s"Expected clustering inflight $clusteringInstantTime in result; got ${result.map(_.getString(0)).mkString(",")}")
+ assertResult(HoodieTimeline.CLUSTERING_ACTION)(clusteringRow.get.getString(1))
+ assertResult(true)(clusteringRow.get.getBoolean(2))
+
+ val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator
+ val expectedInflight = instantGenerator.createNewInstant(
+ HoodieInstant.State.INFLIGHT, HoodieTimeline.CLUSTERING_ACTION, clusteringInstantTime)
+ assert(!metaClient.reloadActiveTimeline().getInstants.contains(expectedInflight),
+ s"Clustering inflight $clusteringInstantTime should be gone after rollback")
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits handles partitioned COW table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ createEmptyPartitionedTable(tableName, tablePath, "cow")
+
+ val staleTs = "20200101120000"
+ injectInflightInstant(tablePath, HoodieTimeline.REPLACE_COMMIT_ACTION, staleTs)
+
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ assertResult(HoodieTimeline.REPLACE_COMMIT_ACTION)(result(0).getString(1))
+ assertResult(true)(result(0).getBoolean(2))
+ }
+ }
+
+ test("Test cleanup_stale_inflight_commits handles MOR table DELTA_COMMIT_ACTION with include_ingestion_commits") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ createEmptyPartitionedTable(tableName, tablePath, "mor")
+
+ val staleTs = "20200101120000"
+ injectInflightInstant(tablePath, HoodieTimeline.DELTA_COMMIT_ACTION, staleTs)
+
+ // Default (include_ingestion_commits=false): DELTA_COMMIT_ACTION is filtered out
+ val defaultResult = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1)").collect()
+ assertResult(0)(defaultResult.length)
+
+ // With include_ingestion_commits=true: DELTA_COMMIT_ACTION is processed
+ val result = spark.sql(
+ s"call cleanup_stale_inflight_commits(table => '$tableName', " +
+ s"allowed_inflight_interval_minutes => 1, " +
+ s"include_ingestion_commits => true)").collect()
+
+ assertResult(1)(result.length)
+ assertResult(staleTs)(result(0).getString(0))
+ assertResult(HoodieTimeline.DELTA_COMMIT_ACTION)(result(0).getString(1))
+ assertResult(true)(result(0).getBoolean(2))
+ }
+ }
+
+ /**
+ * Injects a REQUESTED→INFLIGHT instant into the active timeline without completing it.
+ * Used to simulate stale inflight operations for testing.
+ */
+ private def injectInflightInstant(tablePath: String, action: String, instantTime: String): Unit = {
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val timeline = metaClient.getActiveTimeline
+ val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator
+ val requested = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, action, instantTime)
+ timeline.createNewInstant(requested)
+ timeline.transitionRequestedToInflight(requested, HOption.empty[Array[Byte]]())
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala
new file mode 100644
index 000000000000..ca77172bd607
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.procedure
+
+import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
+
+import java.text.SimpleDateFormat
+import java.util.Date
+
+class TestShowInflightCommitsProcedure extends HoodieSparkProcedureTestBase {
+
+ test("Test show_inflight_commits returns empty for a fully committed table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | ts long
+ | ) using hudi
+ | location '${tmp.getCanonicalPath}'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+ spark.sql(s"insert into $tableName values(1, 'a1', 1000)")
+ spark.sql(s"insert into $tableName values(2, 'a2', 2000)")
+
+ val result = spark.sql(s"call show_inflight_commits(table => '$tableName')").collect()
+ assertResult(0)(result.length)
+ }
+ }
+
+ test("Test show_inflight_commits returns injected inflight instant") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | ts long
+ | ) using hudi
+ | location '$tablePath'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+
+ val injectedTs = "20200101120000"
+ injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, injectedTs)
+
+ val result = spark.sql(s"call show_inflight_commits(table => '$tableName')").collect()
+ assert(result.length >= 1)
+ val row = result.find(r => r.getString(0) == injectedTs)
+ assert(row.isDefined, s"Expected inflight instant $injectedTs not found in results")
+ assertResult(HoodieTimeline.COMMIT_ACTION)(row.get.getString(1))
+ assertResult("INFLIGHT")(row.get.getString(2))
+ }
+ }
+
+ test("Test show_inflight_commits min_age_minutes filter includes old and excludes recent") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | ts long
+ | ) using hudi
+ | location '$tablePath'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+
+ val oldTs = "20200101120000"
+ val freshTs = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date())
+
+ injectInflightInstant(tablePath, HoodieTimeline.COMMIT_ACTION, oldTs)
+ injectInflightInstant(tablePath, HoodieTimeline.DELTA_COMMIT_ACTION, freshTs)
+
+ // No filter: both should appear
+ val allResults = spark.sql(s"call show_inflight_commits(table => '$tableName', min_age_minutes => 0)").collect()
+ assert(allResults.length >= 2)
+ assert(allResults.exists(r => r.getString(0) == oldTs))
+ assert(allResults.exists(r => r.getString(0) == freshTs))
+
+ // 60-minute filter: only the old instant should appear
+ val filteredResults = spark.sql(
+ s"call show_inflight_commits(table => '$tableName', min_age_minutes => 60)").collect()
+ assert(filteredResults.exists(r => r.getString(0) == oldTs),
+ s"Expected old instant $oldTs to appear with min_age_minutes=60")
+ assert(!filteredResults.exists(r => r.getString(0) == freshTs),
+ s"Fresh instant $freshTs should not appear with min_age_minutes=60")
+ }
+ }
+
+ test("Test show_inflight_commits requires table parameter") {
+ checkExceptionContain(
+ "call show_inflight_commits()")(
+ "Argument: table is required")
+ }
+
+ /**
+ * Injects a REQUESTED→INFLIGHT instant into the active timeline without completing it.
+ * Used to simulate stale inflight operations for testing.
+ */
+ private def injectInflightInstant(tablePath: String, action: String, instantTime: String): Unit = {
+ val metaClient = HoodieTableMetaClient.builder
+ .setConf(HadoopFSUtils.getStorageConfWithCopy(spark.sparkContext.hadoopConfiguration))
+ .setBasePath(tablePath)
+ .build
+ val timeline = metaClient.getActiveTimeline
+ val instantGenerator = metaClient.getTimelineLayout.getInstantGenerator
+ val requested = instantGenerator.createNewInstant(HoodieInstant.State.REQUESTED, action, instantTime)
+ timeline.createNewInstant(requested)
+ timeline.transitionRequestedToInflight(requested, HOption.empty[Array[Byte]]())
+ }
+}
(hudi) 09/12: chore: Fix Flink CI Maven profile arguments (#18845)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 3e3d0b8af1ee106cdbf06c28d45e2730b280bb9a
Author: Shuo Cheng <nj...@gmail.com>
AuthorDate: Tue May 26 19:51:11 2026 +0800
chore: Fix Flink CI Maven profile arguments (#18845)
(cherry picked from commit 76ddebcc522876fe557647fa12631fa698ad6263)
---
.github/workflows/bot.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 06468d0c4fa2..fb7dd760c505 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -580,14 +580,14 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
- mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
- name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
SPARK_PROFILE: ${{ matrix.sparkProfile }}
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
- mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false
+ mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false
- name: Generate merged coverage report
if: always()
run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
@@ -1074,7 +1074,7 @@ jobs:
mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
# TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708.
sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources
- mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION"
+ mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION"
fi
- name: IT - Bundle Validation - OpenJDK 11
env:
@@ -1316,7 +1316,7 @@ jobs:
FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
run:
- mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
+ mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
- name: Quickstart Test
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
(hudi) 11/12: docs: Update DOAP file to include 0.15.1 (#18838)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 6834874bedfa600a56f6f8ab6b1a8bbdb891365d
Author: voonhous <vo...@gmail.com>
AuthorDate: Wed May 27 11:05:44 2026 +0800
docs: Update DOAP file to include 0.15.1 (#18838)
(cherry picked from commit ce29423932ec2a4088e08c8fae3a73e1f6d9dcb8)
---
doap_HUDI.rdf | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf
index 921decaeec72..9f56441f3f45 100644
--- a/doap_HUDI.rdf
+++ b/doap_HUDI.rdf
@@ -223,12 +223,21 @@
<created>2025-11-17</created>
<revision>1.1.0</revision>
</Version>
+ </release>
+ <release>
<Version>
<name>Apache Hudi 1.1.1</name>
<created>2025-12-18</created>
<revision>1.1.1</revision>
</Version>
</release>
+ <release>
+ <Version>
+ <name>Apache Hudi 0.15.1</name>
+ <created>2026-05-19</created>
+ <revision>0.15.1</revision>
+ </Version>
+ </release>
<release>
<Version>
<name>Apache Hudi 1.2.0</name>
(hudi) 05/12: chore: Fix stale zip file for variant backward compatibility test (#18815)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 532edf360a4bf4d2e39b747e6e0748c50857bbd0
Author: Shuo Cheng <nj...@gmail.com>
AuthorDate: Tue May 26 10:28:30 2026 +0800
chore: Fix stale zip file for variant backward compatibility test (#18815)
(cherry picked from commit 7e5b5351bbfa4fc09a64b19319c047600360f611)
---
.../variant_backward_compat/variant_mor_avro.zip | Bin 87511 -> 104069 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip b/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip
index 59244af7dd22..610188de288b 100644
Binary files a/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip and b/hudi-common/src/test/resources/variant_backward_compat/variant_mor_avro.zip differ
(hudi) 07/12: fix(spark): Add options for archive procedure (#18437)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c7be772580dab386e569836933c9096b8cd605c0
Author: fhan <aa...@gmail.com>
AuthorDate: Tue May 26 11:18:46 2026 +0800
fix(spark): Add options for archive procedure (#18437)
* fix(spark): Add options for archive procedure
* set 'enable_metadata' default value to true
* fix args in SparkMain
* fix options in ArchiveCommitsProcedure
* fix(spark): set named parameters with higher priority and improve extractOptions()
* optimize entire impl and add UTs for HoodieCLIUtils
* optimize ArchiveCommitsProcedure.
* optimize ArchiveExecutorUtils and HoodieCLIUtils according to hudi-agent review results.
---------
Co-authored-by: fhan <yf...@jd.com>
(cherry picked from commit b5c5801bf7d5e0b53c60ef5d46cea1e2d22cc6d0)
---
.../org/apache/hudi/cli/commands/SparkMain.java | 2 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 60 ++++++++-
.../scala/org/apache/hudi/TestHoodieCLIUtils.scala | 104 +++++++++++++++
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 17 ++-
.../procedures/ArchiveCommitsProcedure.scala | 99 ++++++++++++--
.../procedure/TestArchiveCommitsProcedure.scala | 147 +++++++++++++++------
6 files changed, 372 insertions(+), 57 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
index 097ba984dc92..a7955032497e 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
@@ -599,7 +599,7 @@ public class SparkMain {
private static int archive(JavaSparkContext jsc, int minCommits, int maxCommits, int commitsRetained, boolean enableMetadata, String basePath) {
try {
- return ArchiveExecutorUtils.archive(jsc, minCommits, maxCommits, commitsRetained, enableMetadata, basePath);
+ return ArchiveExecutorUtils.archive(jsc, minCommits, maxCommits, commitsRetained, enableMetadata, basePath, new HashMap<>());
} catch (IOException ex) {
return -1;
}
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala
index 1b40b0fe5701..c6e37f5bb6dd 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCLIUtils.scala
@@ -107,11 +107,63 @@ object HoodieCLIUtils extends Logging {
}
}
+ /**
+ * Parse a comma-separated string of key=value pairs into a Map.
+ *
+ * Notes:
+ * - Whitespace surrounding keys/values is trimmed; empty tokens (e.g. from a
+ * trailing comma or `", ,"`) are silently ignored.
+ * - The delimiter is the first `=` in a token, so values may themselves
+ * contain `=` (e.g. `k=a=b` parses to `k -> "a=b"`).
+ * - Values cannot contain literal commas; the parser does not support
+ * escaping. Configs that need commas should be set via Spark conf instead.
+ * - If the same key appears more than once, a WARN is logged and the last
+ * occurrence wins (consistent with `toMap`'s last-write-wins semantics).
+ *
+ * @throws IllegalArgumentException if a non-empty token does not contain `=`
+ * or has an empty key.
+ */
def extractOptions(s: String): Map[String, String] = {
- StringUtils.split(s, ",").asScala
- .map(split => StringUtils.split(split, "="))
- .map(pair => pair.get(0) -> pair.get(1))
- .toMap
+ if (s == null) {
+ Map.empty
+ } else {
+ // Single pass: build the result Map and collect duplicate keys at the
+ // same time, avoiding an intermediate Seq + groupBy + toMap chain.
+ val (result, duplicates) = StringUtils.split(s, ",").asScala
+ .map(_.trim)
+ .filter(_.nonEmpty)
+ .map(parseOptionToken)
+ .foldLeft((Map.empty[String, String], Set.empty[String])) {
+ case ((acc, dups), (key, value)) =>
+ val newDups = if (acc.contains(key)) dups + key else dups
+ (acc + (key -> value), newDups)
+ }
+
+ if (duplicates.nonEmpty) {
+ logWarning(s"Duplicate option keys detected: ${duplicates.mkString(", ")}. "
+ + "The last occurrence will take effect.")
+ }
+ result
+ }
+ }
+
+ private def parseOptionToken(token: String): (String, String) = {
+ val delimiterIndex = token.indexOf('=')
+ if (delimiterIndex <= 0) {
+ throw new IllegalArgumentException(
+ s"Invalid options format: '$token'. Expected 'key=value' pairs separated by commas, "
+ + "for example: 'k1=v1,k2=v2'.")
+ }
+
+ val key = token.substring(0, delimiterIndex).trim
+ if (key.isEmpty) {
+ throw new IllegalArgumentException(
+ s"Invalid options format: '$token'. Option key must not be empty and options should "
+ + "follow 'key=value' format.")
+ }
+
+ val value = token.substring(delimiterIndex + 1).trim
+ key -> value
}
def getLockOptions(tablePath: String, schema: String, lockConfig: TypedProperties): Map[String, String] = {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala
new file mode 100644
index 000000000000..ca4869286c26
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi
+
+import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue}
+import org.junit.jupiter.api.Test
+
+class TestHoodieCLIUtils {
+
+ @Test
+ def testExtractOptionsBasic(): Unit = {
+ val parsed = HoodieCLIUtils.extractOptions("k1=v1,k2=v2")
+ assertEquals(2, parsed.size)
+ assertEquals("v1", parsed("k1"))
+ assertEquals("v2", parsed("k2"))
+ }
+
+ @Test
+ def testExtractOptionsTrimsWhitespace(): Unit = {
+ val parsed = HoodieCLIUtils.extractOptions(" k1 = v1 , k2= v 2 ")
+ assertEquals("v1", parsed("k1"))
+ // internal whitespace inside value is preserved, only edges are trimmed
+ assertEquals("v 2", parsed("k2"))
+ }
+
+ @Test
+ def testExtractOptionsIgnoresEmptyTokens(): Unit = {
+ // trailing comma, consecutive commas, leading comma — all silently ignored
+ val parsed = HoodieCLIUtils.extractOptions(",k1=v1,, ,k2=v2,")
+ assertEquals(2, parsed.size)
+ assertEquals("v1", parsed("k1"))
+ assertEquals("v2", parsed("k2"))
+ }
+
+ @Test
+ def testExtractOptionsValueContainsEquals(): Unit = {
+ // only the first `=` should be treated as a delimiter
+ val parsed = HoodieCLIUtils.extractOptions("k=a=b=c")
+ assertEquals(1, parsed.size)
+ assertEquals("a=b=c", parsed("k"))
+ }
+
+ @Test
+ def testExtractOptionsAllowsEmptyValue(): Unit = {
+ val parsed = HoodieCLIUtils.extractOptions("k=")
+ assertEquals(1, parsed.size)
+ assertEquals("", parsed("k"))
+ }
+
+ @Test
+ def testExtractOptionsDuplicateKeyLastWins(): Unit = {
+ val parsed = HoodieCLIUtils.extractOptions("k=v1,k=v2,k=v3")
+ assertEquals(1, parsed.size)
+ assertEquals("v3", parsed("k"))
+ }
+
+ @Test
+ def testExtractOptionsNullAndEmpty(): Unit = {
+ assertTrue(HoodieCLIUtils.extractOptions(null).isEmpty)
+ assertTrue(HoodieCLIUtils.extractOptions("").isEmpty)
+ assertTrue(HoodieCLIUtils.extractOptions(" ").isEmpty)
+ assertTrue(HoodieCLIUtils.extractOptions(",,, ").isEmpty)
+ }
+
+ @Test
+ def testExtractOptionsThrowsOnMissingDelimiter(): Unit = {
+ val ex = assertThrows(
+ classOf[IllegalArgumentException],
+ () => HoodieCLIUtils.extractOptions("k1=v1,invalid"))
+ assertTrue(ex.getMessage.contains("invalid"))
+ }
+
+ @Test
+ def testExtractOptionsThrowsOnEmptyKey(): Unit = {
+ val ex = assertThrows(
+ classOf[IllegalArgumentException],
+ () => HoodieCLIUtils.extractOptions("=v"))
+ assertTrue(ex.getMessage.contains("key=value") || ex.getMessage.contains("Option key"))
+ }
+
+ @Test
+ def testExtractOptionsThrowsOnWhitespaceKey(): Unit = {
+ assertThrows(
+ classOf[IllegalArgumentException],
+ () => HoodieCLIUtils.extractOptions(" =v"))
+ }
+}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java
index 772450903e5f..b52d1965e040 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java
+++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java
@@ -40,6 +40,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.spark.api.java.JavaSparkContext;
import java.io.IOException;
+import java.util.Map;
/**
* Archive Utils.
@@ -53,12 +54,26 @@ public final class ArchiveExecutorUtils {
int maxCommits,
int commitsRetained,
boolean enableMetadata,
- String basePath) throws IOException {
+ String basePath,
+ Map<String, String> options) throws IOException {
+ // NOTE on builder ordering:
+ // `withArchivalConfig`/`withCleanConfig`/`withMetadataConfig` each call
+ // `putAll(subConfig.getProps())` onto `writeConfig.getProps()`, which
+ // includes every key filled in by `setDefaults` during the sub-config's
+ // `build()`. If `withProps(conf)` ran BEFORE them, those defaults would
+ // overwrite the user's options (e.g. `hoodie.keep.min.commits`).
+ //
+ // Therefore `withProps(conf)` is intentionally placed LAST so user-supplied
+ // options reliably win over sub-config defaults. Named procedure params
+ // (min/max/retain/enableMetadata) are forwarded via the dedicated builders
+ // below; if the caller wants those to win over a same-name key in `conf`,
+ // the procedure layer is responsible for not putting that key into `conf`.
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath)
.withArchivalConfig(HoodieArchivalConfig.newBuilder().archiveCommitsWith(minCommits, maxCommits).build())
.withCleanConfig(HoodieCleanConfig.newBuilder().retainCommits(commitsRetained).build())
.withEmbeddedTimelineServerEnabled(false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(enableMetadata).build())
+ .withProps(options)
.build();
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
HoodieSparkTable<HoodieAvroPayload> table = HoodieSparkTable.create(config, context);
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala
index efc5a0cc5c2a..6dd578c91cf6 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala
@@ -17,8 +17,10 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.hudi.SparkAdapterSupport
+import org.apache.hudi.{HoodieCLIUtils, SparkAdapterSupport}
import org.apache.hudi.cli.ArchiveExecutorUtils
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -26,17 +28,36 @@ import org.apache.spark.sql.types._
import java.util.function.Supplier
+import scala.collection.JavaConverters._
+
class ArchiveCommitsProcedure extends BaseProcedure
with ProcedureBuilder
with SparkAdapterSupport
with Logging {
+ // NOTE: min_commits / max_commits / retain_commits / enable_metadata are
+ // intentionally declared WITHOUT default values. Whether a caller actually
+ // passed them is determined by `isArgDefined`; their effective values fall
+ // back to the corresponding ConfigProperty defaults (see `call`).
private val PARAMETERS = Array[ProcedureParameter](
ProcedureParameter.optional(0, "table", DataTypes.StringType),
ProcedureParameter.optional(1, "path", DataTypes.StringType),
- ProcedureParameter.optional(2, "min_commits", DataTypes.IntegerType, 20),
- ProcedureParameter.optional(3, "max_commits", DataTypes.IntegerType, 30),
- ProcedureParameter.optional(4, "retain_commits", DataTypes.IntegerType, 10),
- ProcedureParameter.optional(5, "enable_metadata", DataTypes.BooleanType, true)
+ ProcedureParameter.optional(2, "min_commits", DataTypes.IntegerType),
+ ProcedureParameter.optional(3, "max_commits", DataTypes.IntegerType),
+ ProcedureParameter.optional(4, "retain_commits", DataTypes.IntegerType),
+ ProcedureParameter.optional(5, "enable_metadata", DataTypes.BooleanType),
+ // free-form hoodie.* config overrides; format: 'k1=v1,k2=v2'
+ ProcedureParameter.optional(6, "options", DataTypes.StringType)
+ )
+
+ // Mapping of (named parameter -> hoodie.* config key) used both to merge
+ // named-parameter overrides on top of `options` and to back-fill scalar
+ // values fed to ArchiveExecutorUtils. Listed once to keep the named-param
+ // <-> ConfigProperty wiring in a single place.
+ private val NAMED_PARAM_TO_CONFIG_KEY: Seq[(ProcedureParameter, String)] = Seq(
+ PARAMETERS(2) -> HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.key(),
+ PARAMETERS(3) -> HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.key(),
+ PARAMETERS(4) -> HoodieCleanConfig.CLEANER_COMMITS_RETAINED.key(),
+ PARAMETERS(5) -> HoodieMetadataConfig.ENABLE.key()
)
private val OUTPUT_TYPE = new StructType(Array[StructField](
@@ -52,20 +73,74 @@ class ArchiveCommitsProcedure extends BaseProcedure
val tableName = getArgValueOrDefault(args, PARAMETERS(0))
val tablePath = getArgValueOrDefault(args, PARAMETERS(1))
-
- val minCommits = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Int]
- val maxCommits = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Int]
- val retainCommits = getArgValueOrDefault(args, PARAMETERS(4)).get.asInstanceOf[Int]
- val enableMetadata = getArgValueOrDefault(args, PARAMETERS(5)).get.asInstanceOf[Boolean]
+ val confs = getArchiveConfigs(args)
+
+ val minCommits = parseInt(confs,
+ HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.key(),
+ HoodieArchivalConfig.MIN_COMMITS_TO_KEEP.defaultValue())
+ val maxCommits = parseInt(confs,
+ HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.key(),
+ HoodieArchivalConfig.MAX_COMMITS_TO_KEEP.defaultValue())
+ val retainCommits = parseInt(confs,
+ HoodieCleanConfig.CLEANER_COMMITS_RETAINED.key(),
+ HoodieCleanConfig.CLEANER_COMMITS_RETAINED.defaultValue())
+ val enableMetadata = parseBoolean(confs,
+ HoodieMetadataConfig.ENABLE.key(),
+ HoodieMetadataConfig.ENABLE.defaultValue().toString)
val basePath = getBasePath(tableName, tablePath)
-
Seq(Row(ArchiveExecutorUtils.archive(jsc,
minCommits,
maxCommits,
retainCommits,
enableMetadata,
- basePath)))
+ basePath,
+ confs.asJava)))
+ }
+
+ /**
+ * Build the effective hoodie.* config map by overlaying named parameters
+ * (only those the caller explicitly passed) on top of the user `options`
+ * string. Whether a parameter was explicitly passed is decided by
+ * `isArgDefined` rather than by checking the parameter's default, so the
+ * precedence semantics stay correct even if future maintainers add defaults
+ * back to the named parameters.
+ */
+ private def getArchiveConfigs(args: ProcedureArgs): Map[String, String] = {
+ val optionConfs = getArgValueOrDefault(args, PARAMETERS(6))
+ .map(p => HoodieCLIUtils.extractOptions(p.toString))
+ .getOrElse(Map.empty[String, String])
+
+ NAMED_PARAM_TO_CONFIG_KEY.foldLeft(optionConfs) {
+ case (confs, (parameter, configKey)) =>
+ if (isArgDefined(args, parameter)) {
+ confs + (configKey -> getArgValueOrDefault(args, parameter).get.toString)
+ } else {
+ confs
+ }
+ }
+ }
+
+ private def parseInt(confs: Map[String, String], key: String, default: String): Int = {
+ val raw = confs.getOrElse(key, default)
+ try {
+ raw.toInt
+ } catch {
+ case _: NumberFormatException =>
+ throw new IllegalArgumentException(
+ s"Invalid integer value for '$key': '$raw'. Expected a base-10 integer.")
+ }
+ }
+
+ private def parseBoolean(confs: Map[String, String], key: String, default: String): Boolean = {
+ val raw = confs.getOrElse(key, default).trim.toLowerCase
+ raw match {
+ case "true" => true
+ case "false" => false
+ case _ =>
+ throw new IllegalArgumentException(
+ s"Invalid boolean value for '$key': '$raw'. Expected 'true' or 'false'.")
+ }
}
override def build = new ArchiveCommitsProcedure()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala
index c81ffcfb59d6..e5be572eb65f 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala
@@ -21,52 +21,121 @@ package org.apache.spark.sql.hudi.procedure
class TestArchiveCommitsProcedure extends HoodieSparkProcedureTestBase {
- test("Test Call archive_commits Procedure by Table") {
+ /**
+ * Helper: create a fresh COW table at the given location with `numCommits`
+ * insert commits already written. Returns the table name.
+ */
+ private def createTableWithCommits(location: String, numCommits: Int): String = {
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long
+ | ) using hudi
+ | location '$location'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | orderingFields = 'ts',
+ | hoodie.metadata.enable = "false"
+ | )
+ |""".stripMargin)
+
+ (1 to numCommits).foreach { i =>
+ spark.sql(s"insert into $tableName values($i, 'a$i', ${i * 10}, ${i * 1000})")
+ }
+ tableName
+ }
+
+ test("Test Call archive_commits Procedure with named parameters") {
withTempDir { tmp =>
- val tableName = generateTableName
- spark.sql(
- s"""
- |create table $tableName (
- | id int,
- | name string,
- | price double,
- | ts long
- | ) using hudi
- | location '${tmp.getCanonicalPath}'
- | tblproperties (
- | primaryKey = 'id',
- | type = 'cow',
- | orderingFields = 'ts',
- | hoodie.metadata.enable = "false"
- | )
- |""".stripMargin)
-
- spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
- spark.sql(s"insert into $tableName values(2, 'a2', 20, 2000)")
- spark.sql(s"insert into $tableName values(3, 'a3', 30, 3000)")
- spark.sql(s"insert into $tableName values(4, 'a4', 40, 4000)")
- spark.sql(s"insert into $tableName values(5, 'a5', 50, 5000)")
- spark.sql(s"insert into $tableName values(6, 'a6', 60, 6000)")
-
- val result1 = spark.sql(s"call archive_commits(table => '$tableName'" +
- s", min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false)")
+ val tableName = createTableWithCommits(tmp.getCanonicalPath, 6)
+
+ val result = spark.sql(
+ s"call archive_commits(table => '$tableName'," +
+ " min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false)")
.collect()
.map(row => Seq(row.getInt(0)))
- assertResult(1)(result1.length)
- assertResult(0)(result1(0).head)
+ assertResult(1)(result.length)
+ assertResult(0)(result(0).head)
- // collect active commits for table
- val commits = spark.sql(s"""call show_commits(table => '$tableName', limit => 10)""").collect()
- assertResult(2) {
- commits.length
- }
+ val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect()
+ assertResult(2)(commits.length)
+
+ val endTs = commits(0).get(0).toString
+ val archived = spark.sql(
+ s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect()
+ assertResult(4)(archived.length)
+ }
+ }
+
+ test("Test Call archive_commits Procedure driven only by options") {
+ withTempDir { tmp =>
+ val tableName = createTableWithCommits(tmp.getCanonicalPath, 6)
+
+ // No min/max named params — archival behavior must come from `options` alone.
+ // This used to fail (Expected 2, but got 6) because withArchivalConfig#putAll
+ // would overwrite hoodie.keep.min.commits/hoodie.keep.max.commits from
+ // user props with the procedure's named-default min=20/max=30.
+ val result = spark.sql(
+ s"call archive_commits(table => '$tableName'," +
+ " retain_commits => 1," +
+ " options => 'hoodie.keep.min.commits=2,hoodie.keep.max.commits=3," +
+ "hoodie.commits.archival.batch=1,hoodie.metadata.enable=false')")
+ .collect()
+ .map(row => Seq(row.getInt(0)))
+ assertResult(1)(result.length)
+ assertResult(0)(result(0).head)
+
+ val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect()
+ assertResult(2)(commits.length)
- // collect archived commits for table
val endTs = commits(0).get(0).toString
- val archivedCommits = spark.sql(s"""call show_archived_commits(table => '$tableName', end_ts => '$endTs')""").collect()
- assertResult(4) {
- archivedCommits.length
+ val archived = spark.sql(
+ s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect()
+ assertResult(4)(archived.length)
+ }
+ }
+
+ test("Test Call archive_commits Procedure: named parameters override options") {
+ withTempDir { tmp =>
+ val tableName = createTableWithCommits(tmp.getCanonicalPath, 6)
+
+ // options requests min=10/max=20 (would archive nothing for 6 commits),
+ // but named min_commits=2/max_commits=3 must take precedence.
+ val result = spark.sql(
+ s"call archive_commits(table => '$tableName'," +
+ " min_commits => 2, max_commits => 3, retain_commits => 1, enable_metadata => false," +
+ " options => 'hoodie.keep.min.commits=10,hoodie.keep.max.commits=20')")
+ .collect()
+ .map(row => Seq(row.getInt(0)))
+ assertResult(1)(result.length)
+ assertResult(0)(result(0).head)
+
+ val commits = spark.sql(s"call show_commits(table => '$tableName', limit => 10)").collect()
+ // named params won → archival happened, only 2 active commits left
+ assertResult(2)(commits.length)
+
+ val endTs = commits(0).get(0).toString
+ val archived = spark.sql(
+ s"call show_archived_commits(table => '$tableName', end_ts => '$endTs')").collect()
+ assertResult(4)(archived.length)
+ }
+ }
+
+ test("Test Call archive_commits Procedure: invalid options string fails fast") {
+ withTempDir { tmp =>
+ val tableName = createTableWithCommits(tmp.getCanonicalPath, 2)
+
+ val ex = intercept[IllegalArgumentException] {
+ spark.sql(
+ s"call archive_commits(table => '$tableName', options => 'invalid_token')")
+ .collect()
}
+ assert(ex.getMessage.contains("Invalid options format"))
}
}
}
(hudi) 04/12: feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (#18809)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a15bc7bd882775d8316e16ccdb9de18f0d7bfe47
Author: Shihuan Liu <sk...@gmail.com>
AuthorDate: Mon May 25 19:23:15 2026 -0700
feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (#18809)
* feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702)
(cherry picked from commit 06dc09b1fedd6ff5640c95f28c8136e8a78bce98)
---
.../table/format/cow/utils/NestedPositionUtil.java | 4 +-
.../reader/ParquetDataColumnReaderFactory.java | 8 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 688 ++++++++++++---------
.../table/format/cow/utils/BooleanArrayList.java | 69 +++
.../hudi/table/format/cow/utils/IntArrayList.java | 93 +++
.../hudi/table/format/cow/utils/LongArrayList.java | 83 +++
.../table/format/cow/utils/NestedPositionUtil.java | 4 +-
.../format/cow/vector/ColumnarGroupArrayData.java | 179 ------
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 -----
.../cow/vector/HeapArrayGroupColumnVector.java | 53 --
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 75 ++-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++++-
.../cow/vector/position/CollectionPosition.java | 57 ++
.../cow/vector/position/LevelDelegation.java | 43 ++
.../format/cow/vector/position/RowPosition.java | 45 ++
.../cow/vector/reader/ArrayColumnReader.java | 473 --------------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 --
.../format/cow/vector/reader/MapColumnReader.java | 56 --
.../cow/vector/reader/NestedColumnReader.java | 278 +++++++++
.../vector/reader/NestedPrimitiveColumnReader.java | 639 +++++++++++++++++++
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 +++-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 72 +++
.../format/cow/vector/type/ParquetGroupField.java | 59 ++
.../cow/vector/type/ParquetPrimitiveField.java | 55 ++
.../cow/vector/TestHeapColumnVectorAccessors.java | 139 +++++
.../cow/vector/TestParquetDecimalVector.java | 188 ++++++
.../reader/TestParquetDataColumnReaderFactory.java | 272 ++++++++
.../cow/vector/type/TestParquetGroupField.java | 134 ++++
33 files changed, 3078 insertions(+), 1386 deletions(-)
diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
index a1fa3c8fa835..3f2f8976b69b 100644
--- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
+++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
@@ -87,8 +87,8 @@ public class NestedPositionUtil {
*
* @param field field that contains array/map column message include max repetition level and
* definition level.
- * @param definitionLevels int array with each value's repetition level.
- * @param repetitionLevels int array with each value's definition level.
+ * @param definitionLevels int array with each value's definition level.
+ * @param repetitionLevels int array with each value's repetition level.
* @return {@link CollectionPosition} contains collections offset array, length array and isNull
* array.
*/
diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
index 3748e53fc284..1abc6ed56c0d 100644
--- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
+++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
@@ -359,9 +359,9 @@ public final class ParquetDataColumnReaderFactory {
}
private static TimestampData int64ToTimestamp(
- boolean utcTimestamp, long value, ChronoUnit unit) {
+ boolean isUtcTimestamp, long value, ChronoUnit unit) {
Instant instant = Instant.EPOCH.plus(value, unit);
- if (utcTimestamp) {
+ if (isUtcTimestamp) {
return TimestampData.fromInstant(instant);
}
return TimestampData.fromTimestamp(Timestamp.from(instant));
@@ -379,10 +379,10 @@ public final class ParquetDataColumnReaderFactory {
}
private static TimestampData int96ToTimestamp(
- boolean utcTimestamp, long nanosOfDay, int julianDay) {
+ boolean isUtcTimestamp, long nanosOfDay, int julianDay) {
long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND);
- if (utcTimestamp) {
+ if (isUtcTimestamp) {
int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND);
return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond);
} else {
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index ab05beb87459..5468dc86a25a 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,19 +19,18 @@
package org.apache.hudi.table.format.cow;
import org.apache.hudi.common.util.ValidationUtils;
-import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector;
import org.apache.hudi.table.format.cow.vector.HeapArrayVector;
import org.apache.hudi.table.format.cow.vector.HeapDecimalVector;
import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector;
import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector;
-import org.apache.hudi.table.format.cow.vector.reader.ArrayColumnReader;
-import org.apache.hudi.table.format.cow.vector.reader.ArrayGroupReader;
import org.apache.hudi.table.format.cow.vector.reader.EmptyColumnReader;
import org.apache.hudi.table.format.cow.vector.reader.FixedLenBytesColumnReader;
import org.apache.hudi.table.format.cow.vector.reader.Int64TimestampColumnReader;
-import org.apache.hudi.table.format.cow.vector.reader.MapColumnReader;
+import org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader;
import org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader;
-import org.apache.hudi.table.format.cow.vector.reader.RowColumnReader;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField;
import org.apache.flink.core.fs.Path;
import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader;
@@ -64,12 +63,14 @@ import org.apache.flink.table.types.logical.DecimalType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.LogicalTypeFamily;
-import org.apache.flink.table.types.logical.LogicalTypeRoot;
import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.ParquetRuntimeException;
import org.apache.parquet.column.ColumnDescriptor;
@@ -77,12 +78,18 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.io.ColumnIO;
+import org.apache.parquet.io.GroupColumnIO;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.PrimitiveColumnIO;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.InvalidSchemaException;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Date;
@@ -90,25 +97,38 @@ import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import static org.apache.flink.table.utils.DateTimeUtils.toInternal;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.apache.parquet.Preconditions.checkArgument;
+import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
/**
* Util for generating {@link ParquetColumnarRowSplitReader}.
*
- * <p>NOTE: reference from Flink release 1.11.2 {@code ParquetSplitReaderUtil}, modify to support INT64
- * based TIMESTAMP_MILLIS as ConvertedType, should remove when Flink supports that.
+ * <p>Uses the Dremel-style nested reader ported from Apache Flink 2.1 (FLINK-35702). For primitive
+ * top-level columns we keep Hudi's specialized readers — {@link Int64TimestampColumnReader},
+ * {@link FixedLenBytesColumnReader}, and the Hudi {@link HeapDecimalVector} — unchanged. For
+ * nested types (ARRAY / MAP / MULTISET / ROW) we build a {@link ParquetField} tree once per
+ * split via {@link #buildFieldsList(List, List, MessageColumnIO)} and delegate reading to
+ * {@link NestedColumnReader}.
+ *
+ * <p>Schema evolution: missing top-level fields are still handled by the caller
+ * ({@link ParquetColumnarRowSplitReader} patches them with null vectors). Missing fields
+ * <em>inside</em> a Row are handled here — {@link #constructField} returns {@code null} for a
+ * child that isn't physically present, and the corresponding child in the pre-allocated vector
+ * is filled with nulls via {@link #createVectorFromConstant} so the Dremel assembler can
+ * passthrough the slot (see {@link NestedColumnReader#readToVector}).
*/
public class ParquetSplitReaderUtil {
- /**
- * Util for generating partitioned {@link ParquetColumnarRowSplitReader}.
- */
+ /** Util for generating partitioned {@link ParquetColumnarRowSplitReader}. */
public static ParquetColumnarRowSplitReader genPartColumnarRowReader(
boolean utcTimestamp,
boolean caseSensitive,
@@ -125,7 +145,7 @@ public class ParquetSplitReaderUtil {
UnboundRecordFilter recordFilter) throws IOException {
ValidationUtils.checkState(Arrays.stream(selectedFields).noneMatch(x -> x == -1),
- "One or more specified columns does not exist in the hudi table.");
+ "One or more specified columns does not exist in the hudi table.");
List<String> selNonPartNames = Arrays.stream(selectedFields)
.mapToObj(i -> fullFieldNames[i])
@@ -182,10 +202,13 @@ public class ParquetSplitReaderUtil {
return readVector;
}
- private static ColumnVector createVectorFromConstant(
- LogicalType type,
- Object value,
- int batchSize) {
+ /**
+ * Builds a constant-filled column vector for either a partition column (non-null value) or a
+ * missing-column slot (null value). Used both at the batch-generator level for partition
+ * injection and at the row-reader level for fields absent from the Parquet file.
+ */
+ public static ColumnVector createVectorFromConstant(
+ LogicalType type, Object value, int batchSize) {
switch (type.getTypeRoot()) {
case CHAR:
case VARCHAR:
@@ -278,6 +301,7 @@ public class ParquetSplitReaderUtil {
value == null ? null : toInternal((Date) value),
batchSize);
case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
HeapTimestampVector tv = new HeapTimestampVector(batchSize);
if (value == null) {
tv.fillWithNulls();
@@ -286,46 +310,41 @@ public class ParquetSplitReaderUtil {
}
return tv;
case ARRAY:
- ArrayType arrayType = (ArrayType) type;
- if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) {
- HeapArrayGroupColumnVector arrayGroup = new HeapArrayGroupColumnVector(batchSize);
- if (value == null) {
- arrayGroup.fillWithNulls();
- return arrayGroup;
- } else {
- throw new UnsupportedOperationException("Unsupported create array with default value.");
- }
- } else {
- HeapArrayVector arrayVector = new HeapArrayVector(batchSize);
- if (value == null) {
- arrayVector.fillWithNulls();
- return arrayVector;
- } else {
- throw new UnsupportedOperationException("Unsupported create array with default value.");
- }
+ if (value != null) {
+ throw new UnsupportedOperationException("Unsupported create array with default value.");
}
+ HeapArrayVector arrayVector = new HeapArrayVector(batchSize);
+ arrayVector.fillWithNulls();
+ return arrayVector;
case MAP:
- HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null);
- if (value == null) {
- mapVector.fillWithNulls();
- return mapVector;
- } else {
- throw new UnsupportedOperationException("Unsupported create map with default value.");
+ case MULTISET:
+ if (value != null) {
+ throw new UnsupportedOperationException(
+ "Unsupported create " + type.getTypeRoot() + " with default value.");
}
+ HeapMapColumnVector mapVector = new HeapMapColumnVector(batchSize, null, null);
+ mapVector.fillWithNulls();
+ return mapVector;
case ROW:
- HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize);
- if (value == null) {
- rowVector.fillWithNulls();
- return rowVector;
- } else {
+ if (value != null) {
throw new UnsupportedOperationException("Unsupported create row with default value.");
}
+ RowType rowType = (RowType) type;
+ WritableColumnVector[] childVectors = new WritableColumnVector[rowType.getFieldCount()];
+ for (int i = 0; i < childVectors.length; i++) {
+ childVectors[i] =
+ (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize);
+ }
+ HeapRowColumnVector rowVector = new HeapRowColumnVector(batchSize, childVectors);
+ rowVector.fillWithNulls();
+ return rowVector;
default:
throw new UnsupportedOperationException("Unsupported type: " + type);
}
}
- private static List<ColumnDescriptor> filterDescriptors(int depth, Type type, List<ColumnDescriptor> columns) throws ParquetRuntimeException {
+ private static List<ColumnDescriptor> filterDescriptors(
+ int depth, Type type, List<ColumnDescriptor> columns) throws ParquetRuntimeException {
List<ColumnDescriptor> filtered = new ArrayList<>();
for (ColumnDescriptor descriptor : columns) {
if (depth >= descriptor.getPath().length) {
@@ -339,24 +358,61 @@ public class ParquetSplitReaderUtil {
return filtered;
}
+ /**
+ * Creates a {@link ColumnReader} for one top-level requested field. For primitive types the
+ * Hudi-specialized reader path is used. For nested types ({@code ARRAY}, {@code MAP},
+ * {@code MULTISET}, {@code ROW}) the Dremel-style {@link NestedColumnReader} is used, driven by
+ * the supplied pre-built {@link ParquetField} tree.
+ *
+ * @param field the {@link ParquetField} tree for this column, built by
+ * {@link #buildFieldsList(List, List, MessageColumnIO)}. Required (non-null) for nested
+ * types; ignored for primitives.
+ */
+ public static ColumnReader createColumnReader(
+ boolean utcTimestamp,
+ LogicalType fieldType,
+ Type physicalType,
+ List<ColumnDescriptor> descriptors,
+ PageReadStore pages,
+ @Nullable ParquetField field) throws IOException {
+ switch (fieldType.getTypeRoot()) {
+ case ARRAY:
+ case MAP:
+ case MULTISET:
+ case ROW:
+ Preconditions.checkNotNull(
+ field, "ParquetField must be provided for nested type: %s", fieldType);
+ return new NestedColumnReader(utcTimestamp, pages, field);
+ default:
+ return createPrimitiveColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages);
+ }
+ }
+
+ /**
+ * Backward-compat entry point kept for callers that don't project nested types and therefore
+ * never need a {@link ParquetField} tree. Forwards to the {@link ParquetField}-aware overload
+ * with a null field; nested types now go through that overload directly.
+ *
+ * @deprecated use {@link #createColumnReader(boolean, LogicalType, Type, List, PageReadStore,
+ * ParquetField)} so nested types take the Dremel path.
+ */
+ @Deprecated
public static ColumnReader createColumnReader(
boolean utcTimestamp,
LogicalType fieldType,
Type physicalType,
List<ColumnDescriptor> descriptors,
PageReadStore pages) throws IOException {
- return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors,
- pages, 0);
+ return createColumnReader(utcTimestamp, fieldType, physicalType, descriptors, pages, null);
}
- private static ColumnReader createColumnReader(
+ private static ColumnReader createPrimitiveColumnReader(
boolean utcTimestamp,
LogicalType fieldType,
Type physicalType,
List<ColumnDescriptor> columns,
- PageReadStore pages,
- int depth) throws IOException {
- List<ColumnDescriptor> descriptors = filterDescriptors(depth, physicalType, columns);
+ PageReadStore pages) throws IOException {
+ List<ColumnDescriptor> descriptors = filterDescriptors(0, physicalType, columns);
ColumnDescriptor descriptor = descriptors.get(0);
PageReader pageReader = pages.getPageReader(descriptor);
switch (fieldType.getTypeRoot()) {
@@ -392,7 +448,9 @@ public class ParquetSplitReaderUtil {
case INT96:
return new TimestampColumnReader(utcTimestamp, descriptor, pageReader);
default:
- throw new AssertionError();
+ throw new AssertionError(
+ "Unexpected physical type for TIMESTAMP: "
+ + descriptor.getPrimitiveType().getPrimitiveTypeName());
}
case DECIMAL:
switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
@@ -403,106 +461,23 @@ public class ParquetSplitReaderUtil {
case BINARY:
return new BytesColumnReader(descriptor, pageReader);
case FIXED_LEN_BYTE_ARRAY:
- return new FixedLenBytesColumnReader(
- descriptor, pageReader);
+ return new FixedLenBytesColumnReader(descriptor, pageReader);
default:
- throw new AssertionError();
- }
- case ARRAY:
- ArrayType arrayType = (ArrayType) fieldType;
- if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) {
- boolean isThreeLevelList = isThreeLevelList(physicalType);
- // 3-level List structure, drill down 2 level to get type for `element`
- Type elementType = isThreeLevelList
- ? physicalType.asGroupType().getType(0).asGroupType().getType(0)
- : physicalType.asGroupType().getType(0);
- int elementDepth = isThreeLevelList ? depth + 2 : depth + 1;
- return new ArrayGroupReader(createColumnReader(
- utcTimestamp,
- arrayType.getElementType(),
- elementType,
- descriptors,
- pages,
- elementDepth));
- } else {
- return new ArrayColumnReader(
- descriptor,
- pageReader,
- utcTimestamp,
- descriptor.getPrimitiveType(),
- fieldType);
- }
- case MAP:
- MapType mapType = (MapType) fieldType;
- ArrayColumnReader keyReader =
- new ArrayColumnReader(
- descriptor,
- pageReader,
- utcTimestamp,
- descriptor.getPrimitiveType(),
- new ArrayType(mapType.getKeyType()));
- ColumnReader<WritableColumnVector> valueReader;
- if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) {
- valueReader = new ArrayGroupReader(createColumnReader(
- utcTimestamp,
- mapType.getValueType(),
- physicalType.asGroupType().getType(0).asGroupType().getType(1), // Get the value physical type
- descriptors.subList(1, descriptors.size()), // remove the key descriptor
- pages,
- depth + 2)); // increase the depth by 2, because there's a key_value entry in the path
- } else {
- valueReader = new ArrayColumnReader(
- descriptors.get(1),
- pages.getPageReader(descriptors.get(1)),
- utcTimestamp,
- descriptors.get(1).getPrimitiveType(),
- new ArrayType(mapType.getValueType()));
+ throw new AssertionError(
+ "Unexpected physical type for DECIMAL: "
+ + descriptor.getPrimitiveType().getPrimitiveTypeName());
}
- return new MapColumnReader(keyReader, valueReader);
- case ROW:
- RowType rowType = (RowType) fieldType;
- GroupType groupType = physicalType.asGroupType();
- List<ColumnReader> fieldReaders = new ArrayList<>();
- for (int i = 0; i < rowType.getFieldCount(); i++) {
- // schema evolution: read the parquet file with a new extended field name.
- int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType);
- if (fieldIndex < 0) {
- fieldReaders.add(new EmptyColumnReader());
- } else {
- // Check for nested row in array with atomic field type.
-
- // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields.
- // In Parquet, an array of rows is stored as separate arrays for each field.
-
- // Limitations: It won't work for multiple nested arrays and maps.
- // The main problem is that the Flink classes and interface don't follow that pattern.
- if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) {
- fieldReaders.add(
- createColumnReader(
- utcTimestamp,
- new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)),
- groupType.getType(fieldIndex),
- descriptors,
- pages,
- depth + 1));
- } else {
- fieldReaders.add(
- createColumnReader(
- utcTimestamp,
- rowType.getTypeAt(i),
- groupType.getType(fieldIndex),
- descriptors,
- pages,
- depth + 1));
- }
- }
- }
- return new RowColumnReader(fieldReaders);
default:
throw new UnsupportedOperationException(fieldType + " is not supported now.");
}
}
+ /**
+ * Creates the writable column vector that the reader will write into. The returned vector shape
+ * matches {@code fieldType}; for ROW types missing physical fields are slotted with null-filled
+ * vectors (sourced from {@link #createVectorFromConstant}) so that the Dremel assembler in
+ * {@link NestedColumnReader} can pass them through unchanged.
+ */
public static WritableColumnVector createWritableColumnVector(
int batchSize,
LogicalType fieldType,
@@ -523,40 +498,48 @@ public class ParquetSplitReaderUtil {
switch (fieldType.getTypeRoot()) {
case BOOLEAN:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.BOOLEAN,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapBooleanVector(batchSize);
case TINYINT:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapByteVector(batchSize);
case DOUBLE:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.DOUBLE, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.DOUBLE,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapDoubleVector(batchSize);
case FLOAT:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.FLOAT, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.FLOAT,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapFloatVector(batchSize);
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapIntVector(batchSize);
case BIGINT:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.INT64, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.INT64,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapLongVector(batchSize);
case SMALLINT:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.INT32, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.INT32,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapShortVector(batchSize);
case CHAR:
case VARCHAR:
case BINARY:
case VARBINARY:
checkArgument(
- typeName == PrimitiveType.PrimitiveTypeName.BINARY, getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
+ typeName == PrimitiveType.PrimitiveTypeName.BINARY,
+ getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapBytesVector(batchSize);
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
@@ -566,112 +549,64 @@ public class ParquetSplitReaderUtil {
case DECIMAL:
checkArgument(
(typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
- || typeName == PrimitiveType.PrimitiveTypeName.BINARY)
+ || typeName == PrimitiveType.PrimitiveTypeName.BINARY)
&& primitiveType.getOriginalType() == OriginalType.DECIMAL,
getPrimitiveTypeCheckFailureMessage(typeName, fieldType));
return new HeapDecimalVector(batchSize);
case ARRAY:
ArrayType arrayType = (ArrayType) fieldType;
- if (arrayType.getElementType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) {
- boolean isThreeLevelList = isThreeLevelList(physicalType);
- // 3-level List structure, drill down 2 level to get type for `element`
- Type elementType = isThreeLevelList
- ? physicalType.asGroupType().getType(0).asGroupType().getType(0)
- : physicalType.asGroupType().getType(0);
- int elementDepth = isThreeLevelList ? depth + 2 : depth + 1;
- return new HeapArrayGroupColumnVector(
- batchSize,
- createWritableColumnVector(
- batchSize,
- arrayType.getElementType(),
- elementType,
- descriptors,
- elementDepth));
- } else {
- return new HeapArrayVector(
- batchSize,
- createWritableColumnVector(
- batchSize,
- arrayType.getElementType(),
- physicalType,
- descriptors,
- depth));
- }
- case MAP:
+ return new HeapArrayVector(
+ batchSize,
+ createWritableColumnVector(
+ batchSize, arrayType.getElementType(), physicalType, descriptors, depth));
+ case MAP: {
MapType mapType = (MapType) fieldType;
- GroupType repeatedType = physicalType.asGroupType().getType(0).asGroupType();
- // the map column has three level paths.
- WritableColumnVector keyColumnVector = createWritableColumnVector(
+ GroupType repeatedType = unwrapMapRepeatedType(physicalType);
+ return new HeapMapColumnVector(
batchSize,
- new ArrayType(mapType.getKeyType().isNullable(), mapType.getKeyType()),
- repeatedType.getType(0),
- descriptors,
- depth + 2);
- WritableColumnVector valueColumnVector;
- if (mapType.getValueType().isAnyOf(LogicalTypeFamily.CONSTRUCTED)) {
- valueColumnVector = new HeapArrayGroupColumnVector(
- batchSize,
- createWritableColumnVector(
- batchSize,
- mapType.getValueType(),
- repeatedType.getType(1).asGroupType(),
- descriptors,
- depth + 2));
- } else {
- valueColumnVector = createWritableColumnVector(
- batchSize,
- new ArrayType(mapType.getValueType().isNullable(), mapType.getValueType()),
- repeatedType.getType(1),
- descriptors,
- depth + 2);
- }
- return new HeapMapColumnVector(batchSize, keyColumnVector, valueColumnVector);
+ createWritableColumnVector(
+ batchSize, mapType.getKeyType(), repeatedType.getType(0), descriptors, depth + 2),
+ createWritableColumnVector(
+ batchSize, mapType.getValueType(), repeatedType.getType(1), descriptors, depth + 2));
+ }
+ case MULTISET: {
+ MultisetType multisetType = (MultisetType) fieldType;
+ GroupType repeatedType = unwrapMapRepeatedType(physicalType);
+ return new HeapMapColumnVector(
+ batchSize,
+ createWritableColumnVector(
+ batchSize,
+ multisetType.getElementType(),
+ repeatedType.getType(0),
+ descriptors,
+ depth + 2),
+ createWritableColumnVector(
+ batchSize,
+ new IntType(false),
+ repeatedType.getType(1),
+ descriptors,
+ depth + 2));
+ }
case ROW:
RowType rowType = (RowType) fieldType;
GroupType groupType = physicalType.asGroupType();
WritableColumnVector[] columnVectors = new WritableColumnVector[rowType.getFieldCount()];
for (int i = 0; i < columnVectors.length; i++) {
- // schema evolution: read the file with a new extended field name.
int fieldIndex = getFieldIndexInPhysicalType(rowType.getFields().get(i).getName(), groupType);
if (fieldIndex < 0) {
- // Check for nested row in array with atomic field type.
-
- // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields.
- // In Parquet, an array of rows is stored as separate arrays for each field.
-
- // Limitations: It won't work for multiple nested arrays and maps.
- // The main problem is that the Flink classes and interface don't follow that pattern.
- if (groupType.getRepetition().equals(Type.Repetition.REPEATED) && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) {
- columnVectors[i] = (WritableColumnVector) createVectorFromConstant(
- new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)), null, batchSize);
- } else {
- columnVectors[i] = (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize);
- }
+ // Schema evolution: logical field is absent from the Parquet file. Slot a null-filled
+ // vector of the correct shape; NestedColumnReader.readRow will pass it through when the
+ // matching ParquetField child is null.
+ columnVectors[i] =
+ (WritableColumnVector) createVectorFromConstant(rowType.getTypeAt(i), null, batchSize);
} else {
- // Check for nested row in array with atomic field type.
-
- // This is done to meet the Parquet field algorithm that pushes multiplicity and structures down to individual fields.
- // In Parquet, an array of rows is stored as separate arrays for each field.
-
- // Limitations: It won't work for multiple nested arrays and maps.
- // The main problem is that the Flink classes and interface don't follow that pattern.
- if (descriptors.get(fieldIndex).getMaxRepetitionLevel() > 0 && !rowType.getTypeAt(i).is(LogicalTypeRoot.ARRAY)) {
- columnVectors[i] =
- createWritableColumnVector(
- batchSize,
- new ArrayType(rowType.getTypeAt(i).isNullable(), rowType.getTypeAt(i)),
- groupType.getType(fieldIndex),
- descriptors,
- depth + 1);
- } else {
- columnVectors[i] =
- createWritableColumnVector(
- batchSize,
- rowType.getTypeAt(i),
- groupType.getType(fieldIndex),
- descriptors,
- depth + 1);
- }
+ columnVectors[i] =
+ createWritableColumnVector(
+ batchSize,
+ rowType.getTypeAt(i),
+ groupType.getType(fieldIndex),
+ descriptors,
+ depth + 1);
}
}
return new HeapRowColumnVector(batchSize, columnVectors);
@@ -681,56 +616,245 @@ public class ParquetSplitReaderUtil {
}
/**
- * Returns the field index with given physical row type {@code groupType} and field name {@code fieldName}.
- *
- * @return The physical field index or -1 if the field does not exist
+ * Peels one {@code repeated group key_value} wrapper off a MAP / MULTISET physical type, matching
+ * Parquet's canonical 3-level map encoding.
*/
- private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) {
- // get index from fileSchema type, else, return -1
- return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1;
+ private static GroupType unwrapMapRepeatedType(Type physicalType) {
+ return physicalType.asGroupType().getType(0).asGroupType();
}
+ // ------------------------------------------------------------------------------------------
+ // ParquetField tree construction (vendored from Apache Flink 2.1 ParquetSplitReaderUtil)
+ //
+ // The only Hudi-specific divergence is in `constructField`: the ROW branch tolerates children
+ // missing from the Parquet file by emitting a null ParquetField child (upstream throws). This
+ // matches the Hudi schema-evolution contract and is the companion to the null-child branch in
+ // `NestedColumnReader#readRow` and the null-vector slot in `createWritableColumnVector#ROW`.
+ // ------------------------------------------------------------------------------------------
+
/**
- * Check whether the given list type is a three-level list type.
- * <p>
- * <list-repetition> group <name> (LIST) {
- * repeated group list {
- * <element-repetition> <element-type> element;
- * }
- * }
- *
- * @param type list type
- * @return true if the list type is a three-level list type
+ * Builds {@link ParquetField} trees — one per top-level projected logical column — that feed
+ * {@link NestedColumnReader}. The returned list mirrors the input {@code children} positionally;
+ * primitive top-level fields produce {@code null} entries (callers don't need a tree for those).
+ */
+ public static List<ParquetField> buildFieldsList(
+ List<RowType.RowField> children, List<String> fieldNames, MessageColumnIO columnIO) {
+ List<ParquetField> list = new ArrayList<>();
+ for (int i = 0; i < children.size(); i++) {
+ RowType.RowField child = children.get(i);
+ if (isNestedType(child.getType())) {
+ list.add(constructField(child, lookupColumnByName(columnIO, fieldNames.get(i))));
+ } else {
+ list.add(null);
+ }
+ }
+ return list;
+ }
+
+ private static boolean isNestedType(LogicalType type) {
+ return type instanceof RowType
+ || type instanceof ArrayType
+ || type instanceof MapType
+ || type instanceof MultisetType;
+ }
+
+ @Nullable
+ private static ParquetField constructField(RowType.RowField rowField, ColumnIO columnIO) {
+ boolean required = columnIO.getType().getRepetition() == REQUIRED;
+ int repetitionLevel = columnIO.getRepetitionLevel();
+ int definitionLevel = columnIO.getDefinitionLevel();
+ LogicalType type = rowField.getType();
+ String fieldName = rowField.getName();
+ if (type instanceof RowType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ RowType rowType = (RowType) type;
+ List<RowType.RowField> childFields = rowType.getFields();
+ List<ParquetField> fieldsList = new ArrayList<>(childFields.size());
+ for (RowType.RowField childField : childFields) {
+ // Hudi schema evolution: a logical child may be absent from the Parquet file. In that
+ // case we emit a null ParquetField so that NestedColumnReader.readRow passes through the
+ // pre-filled null vector instead of recursing.
+ ColumnIO childIo = lookupColumnByNameOrNull(groupColumnIO, childField.getName());
+ if (childIo == null) {
+ fieldsList.add(null);
+ } else {
+ fieldsList.add(constructField(childField, childIo));
+ }
+ }
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(fieldsList));
+ }
+
+ if (type instanceof MapType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
+ MapType mapType = (MapType) type;
+ ParquetField keyField =
+ constructField(
+ new RowType.RowField("", mapType.getKeyType()), keyValueColumnIO.getChild(0));
+ ParquetField valueField =
+ constructField(
+ new RowType.RowField("", mapType.getValueType()), keyValueColumnIO.getChild(1));
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(Arrays.asList(keyField, valueField)));
+ }
+
+ if (type instanceof MultisetType) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
+ MultisetType multisetType = (MultisetType) type;
+ ParquetField keyField =
+ constructField(
+ new RowType.RowField("", multisetType.getElementType()),
+ keyValueColumnIO.getChild(0));
+ ParquetField valueField =
+ constructField(
+ new RowType.RowField("", new IntType()), keyValueColumnIO.getChild(1));
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.unmodifiableList(Arrays.asList(keyField, valueField)));
+ }
+
+ if (type instanceof ArrayType) {
+ ArrayType arrayType = (ArrayType) type;
+ ColumnIO elementTypeColumnIO;
+ if (columnIO instanceof GroupColumnIO) {
+ GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
+ if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) {
+ while (!Objects.equals(groupColumnIO.getName(), fieldName)) {
+ groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0);
+ }
+ elementTypeColumnIO = groupColumnIO;
+ } else {
+ if (arrayType.getElementType() instanceof RowType) {
+ elementTypeColumnIO = groupColumnIO;
+ } else {
+ elementTypeColumnIO = groupColumnIO.getChild(0);
+ }
+ }
+ } else if (columnIO instanceof PrimitiveColumnIO) {
+ elementTypeColumnIO = columnIO;
+ } else {
+ throw new FlinkRuntimeException(String.format("Unknown ColumnIO, %s", columnIO));
+ }
+
+ ParquetField elementField =
+ constructField(
+ new RowType.RowField("", arrayType.getElementType()),
+ getArrayElementColumn(elementTypeColumnIO));
+ if (repetitionLevel == elementField.getRepetitionLevel()) {
+ repetitionLevel = columnIO.getParent().getRepetitionLevel();
+ }
+ return new ParquetGroupField(
+ type,
+ repetitionLevel,
+ definitionLevel,
+ required,
+ Collections.singletonList(elementField));
+ }
+
+ PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO;
+ return new ParquetPrimitiveField(
+ type, required, primitiveColumnIO.getColumnDescriptor(), primitiveColumnIO.getId());
+ }
+
+ /**
+ * Parquet column names are case-insensitive in Flink's lookup. Matches upstream
+ * {@code ParquetSplitReaderUtil.lookupColumnByName}; throws when absent.
*/
- private static boolean isThreeLevelList(Type type) {
- if (type.isPrimitive()) {
- return false;
+ public static ColumnIO lookupColumnByName(GroupColumnIO groupColumnIO, String columnName) {
+ ColumnIO columnIO = lookupColumnByNameOrNull(groupColumnIO, columnName);
+ if (columnIO != null) {
+ return columnIO;
}
- GroupType groupType = type.asGroupType();
- OriginalType originalType = groupType.getOriginalType();
- return originalType == OriginalType.LIST
- && groupType.getType(0).getName().equals("list");
+ throw new FlinkRuntimeException(
+ "Can not find column io for parquet reader. Column name: " + columnName);
}
/**
- * Construct the error message when primitive type mismatches.
- *
- * @param primitiveType Primitive type
- * @param fieldType Logical field type
- * @return The error message
+ * Case-insensitive column lookup that returns {@code null} when no match is found — the
+ * Hudi-specific companion to {@link #lookupColumnByName}, used by {@link #constructField} to
+ * emit null {@link ParquetField} children for fields absent from the Parquet file.
*/
- private static String getPrimitiveTypeCheckFailureMessage(PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) {
- return String.format("Unexpected type exception. Primitive type: %s. Field type: %s.", primitiveType, fieldType.getTypeRoot().name());
+ @Nullable
+ private static ColumnIO lookupColumnByNameOrNull(
+ GroupColumnIO groupColumnIO, String columnName) {
+ ColumnIO columnIO = groupColumnIO.getChild(columnName);
+ if (columnIO != null) {
+ return columnIO;
+ }
+ for (int i = 0; i < groupColumnIO.getChildrenCount(); i++) {
+ if (groupColumnIO.getChild(i).getName().equalsIgnoreCase(columnName)) {
+ return groupColumnIO.getChild(i);
+ }
+ }
+ return null;
+ }
+
+ public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) {
+ while (groupColumnIO.getChildrenCount() == 1) {
+ groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0);
+ }
+ return groupColumnIO;
+ }
+
+ public static ColumnIO getArrayElementColumn(ColumnIO columnIO) {
+ while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) {
+ columnIO = ((GroupColumnIO) columnIO).getChild(0);
+ }
+
+ // Three-level list: skip the synthetic `element` / `list` wrapper when present.
+ if (columnIO instanceof GroupColumnIO
+ && columnIO.getType().getLogicalTypeAnnotation() == null
+ && ((GroupColumnIO) columnIO).getChildrenCount() == 1
+ && !columnIO.getName().equals("array")
+ && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) {
+ return ((GroupColumnIO) columnIO).getChild(0);
+ }
+ return columnIO;
}
/**
- * Construct the error message when original type mismatches.
+ * Returns the field index with given physical row type {@code groupType} and field name
+ * {@code fieldName}.
*
- * @param originalType Original type
- * @param fieldType Logical field type
- * @return The error message
+ * @return the physical field index or -1 if the field does not exist
+ */
+ private static int getFieldIndexInPhysicalType(String fieldName, GroupType groupType) {
+ return groupType.containsField(fieldName) ? groupType.getFieldIndex(fieldName) : -1;
+ }
+
+ private static String getPrimitiveTypeCheckFailureMessage(
+ PrimitiveType.PrimitiveTypeName primitiveType, LogicalType fieldType) {
+ return String.format(
+ "Unexpected type exception. Primitive type: %s. Field type: %s.",
+ primitiveType, fieldType.getTypeRoot().name());
+ }
+
+ private static String getOriginalTypeCheckFailureMessage(
+ OriginalType originalType, LogicalType fieldType) {
+ return String.format(
+ "Unexpected type exception. Original type: %s. Field type: %s.",
+ originalType, fieldType.getTypeRoot().name());
+ }
+
+ /**
+ * Returns a synthetic null-column reader to fill missing top-level fields. Kept as a convenience
+ * for callers that need to mirror Hudi's original behaviour where a missing column produces an
+ * explicit null-valued reader rather than being omitted from the batch.
*/
- private static String getOriginalTypeCheckFailureMessage(OriginalType originalType, LogicalType fieldType) {
- return String.format("Unexpected type exception. Original type: %s. Field type: %s.", originalType, fieldType.getTypeRoot().name());
+ public static ColumnReader<WritableColumnVector> emptyColumnReader() {
+ return new EmptyColumnReader();
}
}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
new file mode 100644
index 000000000000..d51d7ee754b8
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+
+/**
+ * Minimal implementation of an array-backed list of booleans.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.runtime.util.BooleanArrayList}) because Flink 1.18 does not ship this helper.
+ */
+public class BooleanArrayList {
+ private int size;
+ private boolean[] array;
+
+ public BooleanArrayList(int capacity) {
+ this.size = 0;
+ this.array = new boolean[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(boolean element) {
+ grow(size + 1);
+ array[size++] = element;
+ return true;
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return (size == 0);
+ }
+
+ public boolean[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ private void grow(int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final boolean[] t = new boolean[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
new file mode 100644
index 000000000000..4787dbb5b9dd
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+/**
+ * Minimal implementation of an array-backed list of ints.
+ *
+ * <p>Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.IntArrayList}) to
+ * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader.
+ */
+public class IntArrayList {
+
+ private int size;
+ private int[] array;
+
+ public IntArrayList(final int capacity) {
+ this.size = 0;
+ this.array = new int[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(final int number) {
+ grow(size + 1);
+ array[size++] = number;
+ return true;
+ }
+
+ public int removeLast() {
+ if (size == 0) {
+ throw new NoSuchElementException();
+ }
+ --size;
+ return array[size];
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return size == 0;
+ }
+
+ private void grow(final int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final int[] t = new int[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+
+ public int[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ public static final IntArrayList EMPTY =
+ new IntArrayList(0) {
+
+ @Override
+ public boolean add(int number) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int removeLast() {
+ throw new UnsupportedOperationException();
+ }
+ };
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
new file mode 100644
index 000000000000..a51291f9d844
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.utils;
+
+import java.util.Arrays;
+
+/**
+ * Minimal implementation of an array-backed list of longs.
+ *
+ * <p>Note: Vendored from Apache Flink ({@code org.apache.flink.runtime.util.LongArrayList}) to
+ * avoid depending on {@code @Internal} Flink runtime classes from Hudi's parquet reader.
+ */
+public class LongArrayList {
+
+ private int size;
+ private long[] array;
+
+ public LongArrayList(int capacity) {
+ this.size = 0;
+ this.array = new long[capacity];
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public boolean add(long number) {
+ grow(size + 1);
+ array[size++] = number;
+ return true;
+ }
+
+ public long removeLong(int index) {
+ if (index >= size) {
+ throw new IndexOutOfBoundsException(
+ "Index (" + index + ") is greater than or equal to list size (" + size + ")");
+ }
+ final long old = array[index];
+ size--;
+ if (index != size) {
+ System.arraycopy(array, index + 1, array, index, size - index);
+ }
+ return old;
+ }
+
+ public void clear() {
+ size = 0;
+ }
+
+ public boolean isEmpty() {
+ return (size == 0);
+ }
+
+ public long[] toArray() {
+ return Arrays.copyOf(array, size);
+ }
+
+ private void grow(int length) {
+ if (length > array.length) {
+ final int newLength =
+ (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length);
+ final long[] t = new long[newLength];
+ System.arraycopy(array, 0, t, 0, size);
+ array = t;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
similarity index 98%
copy from hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
copy to hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
index a1fa3c8fa835..3f2f8976b69b 100644
--- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
@@ -87,8 +87,8 @@ public class NestedPositionUtil {
*
* @param field field that contains array/map column message include max repetition level and
* definition level.
- * @param definitionLevels int array with each value's repetition level.
- * @param repetitionLevels int array with each value's definition level.
+ * @param definitionLevels int array with each value's definition level.
+ * @param repetitionLevels int array with each value's repetition level.
* @return {@link CollectionPosition} contains collections offset array, length array and isNull
* array.
*/
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
deleted file mode 100644
index 4c9275f3b093..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector;
-
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-public class ColumnarGroupArrayData implements ArrayData {
-
- WritableColumnVector vector;
- int rowId;
-
- public ColumnarGroupArrayData(WritableColumnVector vector, int rowId) {
- this.vector = vector;
- this.rowId = rowId;
- }
-
- @Override
- public int size() {
- if (vector == null) {
- return 0;
- }
-
- if (vector instanceof HeapRowColumnVector) {
- // assume all fields have the same size
- if (((HeapRowColumnVector) vector).vectors == null || ((HeapRowColumnVector) vector).vectors.length == 0) {
- return 0;
- }
- return ((HeapArrayVector) ((HeapRowColumnVector) vector).vectors[0]).getArray(rowId).size();
- }
- throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector");
- }
-
- @Override
- public boolean isNullAt(int index) {
- if (vector == null) {
- return true;
- }
-
- if (vector instanceof HeapRowColumnVector) {
- return ((HeapRowColumnVector) vector).vectors == null;
- }
-
- throw new UnsupportedOperationException(vector.getClass().getName() + " is not supported. Supported vector types: HeapRowColumnVector");
- }
-
- @Override
- public boolean getBoolean(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public byte getByte(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public short getShort(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public int getInt(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public long getLong(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public float getFloat(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public double getDouble(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public StringData getString(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public DecimalData getDecimal(int index, int precision, int scale) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public TimestampData getTimestamp(int index, int precision) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public <T> RawValueData<T> getRawValue(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public byte[] getBinary(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public ArrayData getArray(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public MapData getMap(int index) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public RowData getRow(int index, int numFields) {
- return new ColumnarGroupRowData((HeapRowColumnVector) vector, rowId, index);
- }
-
- @Override
- public boolean[] toBooleanArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public byte[] toByteArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public short[] toShortArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public int[] toIntArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public long[] toLongArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public float[] toFloatArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public double[] toDoubleArray() {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
deleted file mode 100644
index 69cb6feca13e..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector;
-
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-public class ColumnarGroupMapData implements MapData {
-
- WritableColumnVector keyVector;
- WritableColumnVector valueVector;
- int rowId;
-
- public ColumnarGroupMapData(WritableColumnVector keyVector, WritableColumnVector valueVector, int rowId) {
- this.keyVector = keyVector;
- this.valueVector = valueVector;
- this.rowId = rowId;
- }
-
- @Override
- public int size() {
- if (keyVector == null) {
- return 0;
- }
-
- if (keyVector instanceof HeapArrayVector) {
- return ((HeapArrayVector) keyVector).getArray(rowId).size();
- }
- throw new UnsupportedOperationException(keyVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector");
- }
-
- @Override
- public ArrayData keyArray() {
- return ((HeapArrayVector) keyVector).getArray(rowId);
- }
-
- @Override
- public ArrayData valueArray() {
- if (valueVector instanceof HeapArrayVector) {
- return ((HeapArrayVector) valueVector).getArray(rowId);
- } else if (valueVector instanceof HeapArrayGroupColumnVector) {
- return ((HeapArrayGroupColumnVector) valueVector).getArray(rowId);
- }
- throw new UnsupportedOperationException(valueVector.getClass().getName() + " is not supported. Supported vector types: HeapArrayVector, HeapArrayGroupColumnVector");
- }
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
deleted file mode 100644
index 439c1880823f..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector;
-
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.types.RowKind;
-
-public class ColumnarGroupRowData implements RowData {
-
- HeapRowColumnVector vector;
- int rowId;
- int index;
-
- public ColumnarGroupRowData(HeapRowColumnVector vector, int rowId, int index) {
- this.vector = vector;
- this.rowId = rowId;
- this.index = index;
- }
-
- @Override
- public int getArity() {
- return vector.vectors.length;
- }
-
- @Override
- public RowKind getRowKind() {
- return RowKind.INSERT;
- }
-
- @Override
- public void setRowKind(RowKind rowKind) {
- throw new UnsupportedOperationException("Not support the operation!");
- }
-
- @Override
- public boolean isNullAt(int pos) {
- return
- vector.vectors[pos].isNullAt(rowId)
- || ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).isNullAt(index);
- }
-
- @Override
- public boolean getBoolean(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBoolean(index);
- }
-
- @Override
- public byte getByte(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getByte(index);
- }
-
- @Override
- public short getShort(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getShort(index);
- }
-
- @Override
- public int getInt(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getInt(index);
- }
-
- @Override
- public long getLong(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getLong(index);
- }
-
- @Override
- public float getFloat(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getFloat(index);
- }
-
- @Override
- public double getDouble(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDouble(index);
- }
-
- @Override
- public StringData getString(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getString(index);
- }
-
- @Override
- public DecimalData getDecimal(int pos, int i1, int i2) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getDecimal(index, i1, i2);
- }
-
- @Override
- public TimestampData getTimestamp(int pos, int i1) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getTimestamp(index, i1);
- }
-
- @Override
- public <T> RawValueData<T> getRawValue(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRawValue(index);
- }
-
- @Override
- public byte[] getBinary(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getBinary(index);
- }
-
- @Override
- public ArrayData getArray(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getArray(index);
- }
-
- @Override
- public MapData getMap(int pos) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getMap(index);
- }
-
- @Override
- public RowData getRow(int pos, int numFields) {
- return ((HeapArrayVector) (vector.vectors[pos])).getArray(rowId).getRow(index, numFields);
- }
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
deleted file mode 100644
index 3d7d8b1f0de0..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector;
-
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.columnar.vector.ArrayColumnVector;
-import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-/**
- * This class represents a nullable heap row column vector.
- */
-public class HeapArrayGroupColumnVector extends AbstractHeapVector
- implements WritableColumnVector, ArrayColumnVector {
-
- public WritableColumnVector vector;
-
- public HeapArrayGroupColumnVector(int len) {
- super(len);
- }
-
- public HeapArrayGroupColumnVector(int len, WritableColumnVector vector) {
- super(len);
- this.vector = vector;
- }
-
- @Override
- public ArrayData getArray(int rowId) {
- return new ColumnarGroupArrayData(vector, rowId);
- }
-
- @Override
- public void reset() {
- super.reset();
- vector.reset();
- }
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
index a0dced01e5e8..2f21a323302f 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java
@@ -57,6 +57,37 @@ public class HeapArrayVector extends AbstractHeapVector
return this.isNull.length;
}
+ // ---------------------------------------------------------------------------------------------
+ // Flink 2.1-compatible accessors. Backed by the existing public {@code offsets}, {@code lengths}
+ // and {@code child} fields so legacy callers continue to work; the new {@link
+ // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any
+ // future Flink-2.1-style caller use these accessors.
+ // ---------------------------------------------------------------------------------------------
+
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public void setOffsets(long[] offsets) {
+ this.offsets = offsets;
+ }
+
+ public long[] getLengths() {
+ return lengths;
+ }
+
+ public void setLengths(long[] lengths) {
+ this.lengths = lengths;
+ }
+
+ public ColumnVector getChild() {
+ return child;
+ }
+
+ public void setChild(ColumnVector child) {
+ this.child = child;
+ }
+
@Override
public ArrayData getArray(int i) {
long offset = offsets[i];
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
index 0f088df55c1a..14aad22039e0 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java
@@ -20,30 +20,97 @@ package org.apache.hudi.table.format.cow.vector;
import lombok.Getter;
import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.columnar.ColumnarMapData;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
import org.apache.flink.table.data.columnar.vector.MapColumnVector;
import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
/**
* This class represents a nullable heap map column vector.
+ *
+ * <p>Mirrors {@code org.apache.flink.table.data.columnar.vector.heap.HeapMapVector} from
+ * Flink 2.1 (FLINK-35702). One deliberate divergence from upstream is preserved for backward
+ * compatibility: the {@code keys} / {@code values} fields are typed
+ * {@link WritableColumnVector} rather than upstream's {@link ColumnVector}, so the existing
+ * Lombok-generated {@code getKeys()} / {@code getValues()} accessors keep their original
+ * signature. Callers wanting the Flink-2.1 contract (a {@code ColumnVector}) use
+ * {@link #getKeyColumnVector()} / {@link #getValueColumnVector()}.
*/
public class HeapMapColumnVector extends AbstractHeapVector
implements WritableColumnVector, MapColumnVector {
@Getter
- private final WritableColumnVector keys;
+ private WritableColumnVector keys;
@Getter
- private final WritableColumnVector values;
+ private WritableColumnVector values;
+
+ // ---------------------------------------------------------------------------------------------
+ // Flink 2.1 Dremel-style state. Populated by {@link
+ // org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and
+ // consumed by {@link #getMap(int)}.
+ // ---------------------------------------------------------------------------------------------
+ private long[] offsets;
+ private long[] lengths;
+ private int size;
public HeapMapColumnVector(int len, WritableColumnVector keys, WritableColumnVector values) {
super(len);
+ this.offsets = new long[len];
+ this.lengths = new long[len];
this.keys = keys;
this.values = values;
}
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public void setOffsets(long[] offsets) {
+ this.offsets = offsets;
+ }
+
+ public long[] getLengths() {
+ return lengths;
+ }
+
+ public void setLengths(long[] lengths) {
+ this.lengths = lengths;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public void setSize(int size) {
+ this.size = size;
+ }
+
+ public void setKeys(WritableColumnVector keys) {
+ this.keys = keys;
+ }
+
+ public void setValues(WritableColumnVector values) {
+ this.values = values;
+ }
+
+ /**
+ * Returns the keys child vector typed as {@link ColumnVector}, matching the Flink 2.1 contract
+ * consumed by {@code NestedColumnReader}. Functionally equivalent to {@link #getKeys()}.
+ */
+ public ColumnVector getKeyColumnVector() {
+ return keys;
+ }
+
+ /** Counterpart of {@link #getKeyColumnVector()} for the values child vector. */
+ public ColumnVector getValueColumnVector() {
+ return values;
+ }
+
@Override
public MapData getMap(int rowId) {
- return new ColumnarGroupMapData(keys, values, rowId);
+ long offset = offsets[rowId];
+ long length = lengths[rowId];
+ return new ColumnarMapData(keys, values, (int) offset, (int) length);
}
}
-
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
index ae194e4e6ab0..0c640ce92ee4 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java
@@ -37,6 +37,21 @@ public class HeapRowColumnVector extends AbstractHeapVector
this.vectors = vectors;
}
+ /**
+ * Flink 2.1-compatible accessor for the children vectors. Backed by the existing public {@code
+ * vectors} field so legacy callers continue to work; the new {@link
+ * org.apache.hudi.table.format.cow.vector.reader.NestedColumnReader} (FLINK-35702 port) and any
+ * future Flink-2.1-style caller use this accessor.
+ */
+ public WritableColumnVector[] getFields() {
+ return vectors;
+ }
+
+ /** Counterpart of {@link #getFields()}. */
+ public void setFields(WritableColumnVector[] fields) {
+ this.vectors = fields;
+ }
+
@Override
public ColumnarRowData getRow(int i) {
ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(vectors));
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
index 98b5e6105089..a37b88352cf5 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java
@@ -18,21 +18,29 @@
package org.apache.hudi.table.format.cow.vector;
+import org.apache.flink.formats.parquet.utils.ParquetSchemaConverter;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.columnar.vector.BytesColumnVector;
import org.apache.flink.table.data.columnar.vector.ColumnVector;
import org.apache.flink.table.data.columnar.vector.DecimalColumnVector;
+import org.apache.flink.table.data.columnar.vector.Dictionary;
+import org.apache.flink.table.data.columnar.vector.IntColumnVector;
+import org.apache.flink.table.data.columnar.vector.LongColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableBytesVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableIntVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableLongVector;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
- * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to
- * provide {@link DecimalColumnVector} interface.
- *
- * <p>Reference Flink release 1.11.2 {@link org.apache.flink.formats.parquet.vector.ParquetDecimalVector}
- * because it is not public.
+ * Parquet write decimal as int32 and int64 and binary, this class wrap the real vector to provide
+ * {@link DecimalColumnVector} interface.
*/
-public class ParquetDecimalVector implements DecimalColumnVector {
+public class ParquetDecimalVector
+ implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector {
- public final ColumnVector vector;
+ private final ColumnVector vector;
public ParquetDecimalVector(ColumnVector vector) {
this.vector = vector;
@@ -40,15 +48,180 @@ public class ParquetDecimalVector implements DecimalColumnVector {
@Override
public DecimalData getDecimal(int i, int precision, int scale) {
- return DecimalData.fromUnscaledBytes(
- ((BytesColumnVector) vector).getBytes(i).getBytes(),
- precision,
- scale);
+ if (ParquetSchemaConverter.is32BitDecimal(precision) && vector instanceof IntColumnVector) {
+ return DecimalData.fromUnscaledLong(((IntColumnVector) vector).getInt(i), precision, scale);
+ } else if (ParquetSchemaConverter.is64BitDecimal(precision)
+ && vector instanceof LongColumnVector) {
+ return DecimalData.fromUnscaledLong(((LongColumnVector) vector).getLong(i), precision, scale);
+ } else {
+ checkArgument(
+ vector instanceof BytesColumnVector,
+ "Reading decimal type occur unsupported vector type: %s",
+ vector.getClass());
+ return DecimalData.fromUnscaledBytes(
+ ((BytesColumnVector) vector).getBytes(i).getBytes(), precision, scale);
+ }
+ }
+
+ public ColumnVector getVector() {
+ return vector;
}
@Override
public boolean isNullAt(int i) {
return vector.isNullAt(i);
}
-}
+ @Override
+ public void reset() {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).reset();
+ }
+ }
+
+ @Override
+ public void setNullAt(int rowId) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setNullAt(rowId);
+ }
+ }
+
+ @Override
+ public void setNulls(int rowId, int count) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setNulls(rowId, count);
+ }
+ }
+
+ @Override
+ public void fillWithNulls() {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).fillWithNulls();
+ }
+ }
+
+ @Override
+ public void setDictionary(Dictionary dictionary) {
+ if (vector instanceof WritableColumnVector) {
+ ((WritableColumnVector) vector).setDictionary(dictionary);
+ }
+ }
+
+ @Override
+ public boolean hasDictionary() {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).hasDictionary();
+ }
+ return false;
+ }
+
+ @Override
+ public WritableIntVector reserveDictionaryIds(int capacity) {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).reserveDictionaryIds(capacity);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public WritableIntVector getDictionaryIds() {
+ if (vector instanceof WritableColumnVector) {
+ return ((WritableColumnVector) vector).getDictionaryIds();
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public Bytes getBytes(int i) {
+ if (vector instanceof WritableBytesVector) {
+ return ((WritableBytesVector) vector).getBytes(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void appendBytes(int rowId, byte[] value, int offset, int length) {
+ if (vector instanceof WritableBytesVector) {
+ ((WritableBytesVector) vector).appendBytes(rowId, value, offset, length);
+ }
+ }
+
+ @Override
+ public void fill(byte[] value) {
+ if (vector instanceof WritableBytesVector) {
+ ((WritableBytesVector) vector).fill(value);
+ }
+ }
+
+ @Override
+ public int getInt(int i) {
+ if (vector instanceof WritableIntVector) {
+ return ((WritableIntVector) vector).getInt(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void setInt(int rowId, int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInt(rowId, value);
+ }
+ }
+
+ @Override
+ public void setIntsFromBinary(int rowId, int count, byte[] src, int srcIndex) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setIntsFromBinary(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void setInts(int rowId, int count, int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInts(rowId, count, value);
+ }
+ }
+
+ @Override
+ public void setInts(int rowId, int count, int[] src, int srcIndex) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).setInts(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void fill(int value) {
+ if (vector instanceof WritableIntVector) {
+ ((WritableIntVector) vector).fill(value);
+ }
+ }
+
+ @Override
+ public long getLong(int i) {
+ if (vector instanceof WritableLongVector) {
+ return ((WritableLongVector) vector).getLong(i);
+ }
+ throw new RuntimeException("Child vector must be instance of WritableColumnVector");
+ }
+
+ @Override
+ public void setLong(int rowId, long value) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).setLong(rowId, value);
+ }
+ }
+
+ @Override
+ public void setLongsFromBinary(int rowId, int count, byte[] src, int srcIndex) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).setLongsFromBinary(rowId, count, src, srcIndex);
+ }
+ }
+
+ @Override
+ public void fill(long value) {
+ if (vector instanceof WritableLongVector) {
+ ((WritableLongVector) vector).fill(value);
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
new file mode 100644
index 000000000000..fcdedfbc9d71
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+import javax.annotation.Nullable;
+
+/**
+ * To represent collection's position in repeated type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.CollectionPosition}).
+ */
+public class CollectionPosition {
+ @Nullable private final boolean[] isNull;
+ private final long[] offsets;
+ private final long[] length;
+ private final int valueCount;
+
+ public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) {
+ this.isNull = isNull;
+ this.offsets = offsets;
+ this.length = length;
+ this.valueCount = valueCount;
+ }
+
+ public boolean[] getIsNull() {
+ return isNull;
+ }
+
+ public long[] getOffsets() {
+ return offsets;
+ }
+
+ public long[] getLength() {
+ return length;
+ }
+
+ public int getValueCount() {
+ return valueCount;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
new file mode 100644
index 000000000000..fe95419ac321
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+/**
+ * To delegate repetition level and definition level.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.LevelDelegation}).
+ */
+public class LevelDelegation {
+ private final int[] repetitionLevel;
+ private final int[] definitionLevel;
+
+ public LevelDelegation(int[] repetitionLevel, int[] definitionLevel) {
+ this.repetitionLevel = repetitionLevel;
+ this.definitionLevel = definitionLevel;
+ }
+
+ public int[] getRepetitionLevel() {
+ return repetitionLevel;
+ }
+
+ public int[] getDefinitionLevel() {
+ return definitionLevel;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
new file mode 100644
index 000000000000..5438b6797323
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.position;
+
+import javax.annotation.Nullable;
+
+/**
+ * To represent struct's position in repeated type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.position.RowPosition}).
+ */
+public class RowPosition {
+ @Nullable private final boolean[] isNull;
+ private final int positionsCount;
+
+ public RowPosition(boolean[] isNull, int positionsCount) {
+ this.isNull = isNull;
+ this.positionsCount = positionsCount;
+ }
+
+ public boolean[] getIsNull() {
+ return isNull;
+ }
+
+ public int getPositionsCount() {
+ return positionsCount;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
deleted file mode 100644
index 6a8a01b74946..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector.reader;
-
-import org.apache.hudi.table.format.cow.vector.HeapArrayVector;
-import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
-
-import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch;
-import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
-import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.page.PageReader;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Array {@link ColumnReader}.
- */
-public class ArrayColumnReader extends BaseVectorizedColumnReader {
-
- // The value read in last time
- private Object lastValue;
-
- // flag to indicate if there is no data in parquet data page
- private boolean eof = false;
-
- // flag to indicate if it's the first time to read parquet data page with this instance
- boolean isFirstRow = true;
-
- public ArrayColumnReader(
- ColumnDescriptor descriptor,
- PageReader pageReader,
- boolean isUtcTimestamp,
- Type type,
- LogicalType logicalType)
- throws IOException {
- super(descriptor, pageReader, isUtcTimestamp, type, logicalType);
- }
-
- @Override
- public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
- HeapArrayVector lcv = (HeapArrayVector) vector;
- // before readBatch, initial the size of offsets & lengths as the default value,
- // the actual size will be assigned in setChildrenInfo() after reading complete.
- lcv.offsets = new long[VectorizedColumnBatch.DEFAULT_SIZE];
- lcv.lengths = new long[VectorizedColumnBatch.DEFAULT_SIZE];
- // Because the length of ListColumnVector.child can't be known now,
- // the valueList will save all data for ListColumnVector temporary.
- List<Object> valueList = new ArrayList<>();
-
- LogicalType category = ((ArrayType) logicalType).getElementType();
-
- // read the first row in parquet data page, this will be only happened once for this
- // instance
- if (isFirstRow) {
- if (!fetchNextValue(category)) {
- return;
- }
- isFirstRow = false;
- }
-
- int index = collectDataFromParquetPage(readNumber, lcv, valueList, category);
-
- // Convert valueList to array for the ListColumnVector.child
- fillColumnVector(category, lcv, valueList, index);
- }
-
- /**
- * Reads a single value from parquet page, puts it into lastValue. Returns a boolean indicating
- * if there is more values to read (true).
- *
- * @param category
- * @return boolean
- * @throws IOException
- */
- private boolean fetchNextValue(LogicalType category) throws IOException {
- int left = readPageIfNeed();
- if (left > 0) {
- // get the values of repetition and definitionLevel
- readRepetitionAndDefinitionLevels();
- // read the data if it isn't null
- if (definitionLevel == maxDefLevel) {
- if (isCurrentPageDictionaryEncoded) {
- lastValue = dataColumn.readValueDictionaryId();
- } else {
- lastValue = readPrimitiveTypedRow(category);
- }
- } else {
- lastValue = null;
- }
- return true;
- } else {
- eof = true;
- return false;
- }
- }
-
- private int readPageIfNeed() throws IOException {
- // Compute the number of values we want to read in this page.
- int leftInPage = (int) (endOfPageValueCount - valuesRead);
- if (leftInPage == 0) {
- // no data left in current page, load data from new page
- readPage();
- leftInPage = (int) (endOfPageValueCount - valuesRead);
- }
- return leftInPage;
- }
-
- // Need to be in consistent with that VectorizedPrimitiveColumnReader#readBatchHelper
- // TODO Reduce the duplicated code
- private Object readPrimitiveTypedRow(LogicalType category) {
- switch (category.getTypeRoot()) {
- case CHAR:
- case VARCHAR:
- case BINARY:
- case VARBINARY:
- return dataColumn.readString();
- case BOOLEAN:
- return dataColumn.readBoolean();
- case TIME_WITHOUT_TIME_ZONE:
- case DATE:
- case INTEGER:
- return dataColumn.readInteger();
- case TINYINT:
- return dataColumn.readTinyInt();
- case SMALLINT:
- return dataColumn.readSmallInt();
- case BIGINT:
- return dataColumn.readLong();
- case FLOAT:
- return dataColumn.readFloat();
- case DOUBLE:
- return dataColumn.readDouble();
- case DECIMAL:
- switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
- case INT32:
- return dataColumn.readInteger();
- case INT64:
- return dataColumn.readLong();
- case BINARY:
- case FIXED_LEN_BYTE_ARRAY:
- return dataColumn.readString();
- default:
- throw new AssertionError();
- }
- case TIMESTAMP_WITHOUT_TIME_ZONE:
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return dataColumn.readTimestamp();
- default:
- throw new RuntimeException("Unsupported type in the list: " + type);
- }
- }
-
- private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) {
- if (dictionaryValue == null) {
- return null;
- }
-
- switch (category.getTypeRoot()) {
- case CHAR:
- case VARCHAR:
- case BINARY:
- case VARBINARY:
- return dictionary.readString(dictionaryValue);
- case DATE:
- case TIME_WITHOUT_TIME_ZONE:
- case INTEGER:
- return dictionary.readInteger(dictionaryValue);
- case BOOLEAN:
- return dictionary.readBoolean(dictionaryValue) ? 1 : 0;
- case DOUBLE:
- return dictionary.readDouble(dictionaryValue);
- case FLOAT:
- return dictionary.readFloat(dictionaryValue);
- case TINYINT:
- return dictionary.readTinyInt(dictionaryValue);
- case SMALLINT:
- return dictionary.readSmallInt(dictionaryValue);
- case BIGINT:
- return dictionary.readLong(dictionaryValue);
- case DECIMAL:
- switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
- case INT32:
- return dictionary.readInteger(dictionaryValue);
- case INT64:
- return dictionary.readLong(dictionaryValue);
- case FIXED_LEN_BYTE_ARRAY:
- case BINARY:
- return dictionary.readString(dictionaryValue);
- default:
- throw new AssertionError();
- }
- case TIMESTAMP_WITHOUT_TIME_ZONE:
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return dictionary.readTimestamp(dictionaryValue);
- default:
- throw new RuntimeException("Unsupported type in the list: " + type);
- }
- }
-
- /**
- * Collects data from a parquet page and returns the final row index where it stopped. The
- * returned index can be equal to or less than total.
- *
- * @param total maximum number of rows to collect
- * @param lcv column vector to do initial setup in data collection time
- * @param valueList collection of values that will be fed into the vector later
- * @param category
- * @return int
- * @throws IOException
- */
- private int collectDataFromParquetPage(
- int total, HeapArrayVector lcv, List<Object> valueList, LogicalType category)
- throws IOException {
- int index = 0;
- /*
- * Here is a nested loop for collecting all values from a parquet page.
- * A column of array type can be considered as a list of lists, so the two loops are as below:
- * 1. The outer loop iterates on rows (index is a row index, so points to a row in the batch), e.g.:
- * [0, 2, 3] <- index: 0
- * [NULL, 3, 4] <- index: 1
- *
- * 2. The inner loop iterates on values within a row (sets all data from parquet data page
- * for an element in ListColumnVector), so fetchNextValue returns values one-by-one:
- * 0, 2, 3, NULL, 3, 4
- *
- * As described below, the repetition level (repetitionLevel != 0)
- * can be used to decide when we'll start to read values for the next list.
- */
- while (!eof && index < total) {
- // add element to ListColumnVector one by one
- lcv.offsets[index] = valueList.size();
- /*
- * Let's collect all values for a single list.
- * Repetition level = 0 means that a new list started there in the parquet page,
- * in that case, let's exit from the loop, and start to collect value for a new list.
- */
- do {
- /*
- * Definition level = 0 when a NULL value was returned instead of a list
- * (this is not the same as a NULL value in of a list).
- */
- if (definitionLevel == 0) {
- lcv.setNullAt(index);
- }
- valueList.add(
- isCurrentPageDictionaryEncoded
- ? dictionaryDecodeValue(category, (Integer) lastValue)
- : lastValue);
- } while (fetchNextValue(category) && (repetitionLevel != 0));
-
- lcv.lengths[index] = valueList.size() - lcv.offsets[index];
- index++;
- }
- return index;
- }
-
- /**
- * The lengths & offsets will be initialized as default size (1024), it should be set to the
- * actual size according to the element number.
- */
- private void setChildrenInfo(HeapArrayVector lcv, int itemNum, int elementNum) {
- lcv.setSize(itemNum);
- long[] lcvLength = new long[elementNum];
- long[] lcvOffset = new long[elementNum];
- System.arraycopy(lcv.lengths, 0, lcvLength, 0, elementNum);
- System.arraycopy(lcv.offsets, 0, lcvOffset, 0, elementNum);
- lcv.lengths = lcvLength;
- lcv.offsets = lcvOffset;
- }
-
- private void fillColumnVector(
- LogicalType category, HeapArrayVector lcv, List valueList, int elementNum) {
- int total = valueList.size();
- setChildrenInfo(lcv, total, elementNum);
- switch (category.getTypeRoot()) {
- case CHAR:
- case VARCHAR:
- case BINARY:
- case VARBINARY:
- lcv.child = new HeapBytesVector(total);
- ((HeapBytesVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- byte[] src = ((List<byte[]>) valueList).get(i);
- if (src == null) {
- ((HeapBytesVector) lcv.child).setNullAt(i);
- } else {
- ((HeapBytesVector) lcv.child).appendBytes(i, src, 0, src.length);
- }
- }
- break;
- case BOOLEAN:
- lcv.child = new HeapBooleanVector(total);
- ((HeapBooleanVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapBooleanVector) lcv.child).setNullAt(i);
- } else {
- ((HeapBooleanVector) lcv.child).vector[i] =
- ((List<Boolean>) valueList).get(i);
- }
- }
- break;
- case TINYINT:
- lcv.child = new HeapByteVector(total);
- ((HeapByteVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapByteVector) lcv.child).setNullAt(i);
- } else {
- ((HeapByteVector) lcv.child).vector[i] =
- (byte) ((List<Integer>) valueList).get(i).intValue();
- }
- }
- break;
- case SMALLINT:
- lcv.child = new HeapShortVector(total);
- ((HeapShortVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapShortVector) lcv.child).setNullAt(i);
- } else {
- ((HeapShortVector) lcv.child).vector[i] =
- (short) ((List<Integer>) valueList).get(i).intValue();
- }
- }
- break;
- case INTEGER:
- case DATE:
- case TIME_WITHOUT_TIME_ZONE:
- lcv.child = new HeapIntVector(total);
- ((HeapIntVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapIntVector) lcv.child).setNullAt(i);
- } else {
- ((HeapIntVector) lcv.child).vector[i] = ((List<Integer>) valueList).get(i);
- }
- }
- break;
- case FLOAT:
- lcv.child = new HeapFloatVector(total);
- ((HeapFloatVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapFloatVector) lcv.child).setNullAt(i);
- } else {
- ((HeapFloatVector) lcv.child).vector[i] = ((List<Float>) valueList).get(i);
- }
- }
- break;
- case BIGINT:
- lcv.child = new HeapLongVector(total);
- ((HeapLongVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapLongVector) lcv.child).setNullAt(i);
- } else {
- ((HeapLongVector) lcv.child).vector[i] = ((List<Long>) valueList).get(i);
- }
- }
- break;
- case DOUBLE:
- lcv.child = new HeapDoubleVector(total);
- ((HeapDoubleVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapDoubleVector) lcv.child).setNullAt(i);
- } else {
- ((HeapDoubleVector) lcv.child).vector[i] =
- ((List<Double>) valueList).get(i);
- }
- }
- break;
- case TIMESTAMP_WITHOUT_TIME_ZONE:
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- lcv.child = new HeapTimestampVector(total);
- ((HeapTimestampVector) lcv.child).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapTimestampVector) lcv.child).setNullAt(i);
- } else {
- ((HeapTimestampVector) lcv.child)
- .setTimestamp(i, ((List<TimestampData>) valueList).get(i));
- }
- }
- break;
- case DECIMAL:
- PrimitiveType.PrimitiveTypeName primitiveTypeName =
- descriptor.getPrimitiveType().getPrimitiveTypeName();
- switch (primitiveTypeName) {
- case INT32:
- lcv.child = new ParquetDecimalVector(new HeapIntVector(total));
- ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector)
- .setNullAt(i);
- } else {
- ((HeapIntVector) ((ParquetDecimalVector) lcv.child).vector)
- .vector[i] =
- ((List<Integer>) valueList).get(i);
- }
- }
- break;
- case INT64:
- lcv.child = new ParquetDecimalVector(new HeapLongVector(total));
- ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector).reset();
- for (int i = 0; i < valueList.size(); i++) {
- if (valueList.get(i) == null) {
- ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector)
- .setNullAt(i);
- } else {
- ((HeapLongVector) ((ParquetDecimalVector) lcv.child).vector)
- .vector[i] =
- ((List<Long>) valueList).get(i);
- }
- }
- break;
- default:
- lcv.child = new ParquetDecimalVector(new HeapBytesVector(total));
- ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector).reset();
- for (int i = 0; i < valueList.size(); i++) {
- byte[] src = ((List<byte[]>) valueList).get(i);
- if (valueList.get(i) == null) {
- ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector)
- .setNullAt(i);
- } else {
- ((HeapBytesVector) ((ParquetDecimalVector) lcv.child).vector)
- .appendBytes(i, src, 0, src.length);
- }
- }
- break;
- }
- break;
- default:
- throw new RuntimeException("Unsupported type in the list: " + type);
- }
- }
-}
-
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
deleted file mode 100644
index df7c5d85bc4a..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector.reader;
-
-import org.apache.hudi.table.format.cow.vector.HeapArrayGroupColumnVector;
-
-import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-import java.io.IOException;
-
-/**
- * Array of a Group type (Array, Map, Row, etc.) {@link ColumnReader}.
- */
-public class ArrayGroupReader implements ColumnReader<WritableColumnVector> {
-
- private final ColumnReader<WritableColumnVector> fieldReader;
-
- public ArrayGroupReader(ColumnReader<WritableColumnVector> fieldReader) {
- this.fieldReader = fieldReader;
- }
-
- @Override
- public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
- HeapArrayGroupColumnVector rowColumnVector = (HeapArrayGroupColumnVector) vector;
-
- fieldReader.readToVector(readNumber, rowColumnVector.vector);
- }
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
deleted file mode 100644
index ee65dd22c436..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector.reader;
-
-import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector;
-
-import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
-import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-import java.io.IOException;
-
-/**
- * Map {@link ColumnReader}.
- */
-public class MapColumnReader implements ColumnReader<WritableColumnVector> {
-
- private final ArrayColumnReader keyReader;
- private final ColumnReader<WritableColumnVector> valueReader;
-
- public MapColumnReader(
- ArrayColumnReader keyReader, ColumnReader<WritableColumnVector> valueReader) {
- this.keyReader = keyReader;
- this.valueReader = valueReader;
- }
-
- @Override
- public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
- HeapMapColumnVector mapColumnVector = (HeapMapColumnVector) vector;
- AbstractHeapVector keyArrayColumnVector = (AbstractHeapVector) (mapColumnVector.getKeys());
- keyReader.readToVector(readNumber, mapColumnVector.getKeys());
- valueReader.readToVector(readNumber, mapColumnVector.getValues());
- for (int i = 0; i < keyArrayColumnVector.getLen(); i++) {
- if (keyArrayColumnVector.isNullAt(i)) {
- mapColumnVector.setNullAt(i);
- }
- }
- }
-}
-
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
new file mode 100644
index 000000000000..b4f04b20c477
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.hudi.table.format.cow.utils.NestedPositionUtil;
+import org.apache.hudi.table.format.cow.vector.HeapArrayVector;
+import org.apache.hudi.table.format.cow.vector.HeapMapColumnVector;
+import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector;
+import org.apache.hudi.table.format.cow.vector.position.CollectionPosition;
+import org.apache.hudi.table.format.cow.vector.position.LevelDelegation;
+import org.apache.hudi.table.format.cow.vector.position.RowPosition;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetGroupField;
+import org.apache.hudi.table.format.cow.vector.type.ParquetPrimitiveField;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.AbstractHeapVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.MultisetType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.page.PageReadStore;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ColumnReader used to read a {@code Group} type in Parquet ({@code Map}, {@code Array}, {@code
+ * Row}). Resolves nested structures using Dremel striping/assembly; see <a
+ * href="https://github.com/julienledem/redelm/wiki/The-striping-and-assembly-algorithms-from-the-Dremel-paper">the
+ * striping and assembly algorithms from the Dremel paper</a>.
+ *
+ * <p>Vendored from Apache Flink 2.1 (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.reader.NestedColumnReader}). Differences vs. upstream:
+ *
+ * <ul>
+ * <li>Uses Hudi-local {@code HeapRowColumnVector}/{@code HeapMapColumnVector}/{@code
+ * HeapArrayVector} instead of the Flink-private {@code HeapRowVector}/{@code
+ * HeapMapVector}/{@code HeapArrayVector}.
+ * <li>Supports Hudi's schema-evolution contract: a {@code ParquetGroupField} representing a
+ * {@link RowType} may contain {@code null} children — meaning the corresponding logical
+ * field is absent from the Parquet file. Those slots are passed through unchanged and do
+ * not contribute to the row's repetition/definition-level stream.
+ * </ul>
+ */
+public class NestedColumnReader implements ColumnReader<WritableColumnVector> {
+
+ private final Map<ColumnDescriptor, NestedPrimitiveColumnReader> columnReaders;
+ private final boolean isUtcTimestamp;
+
+ private final PageReadStore pages;
+
+ private final ParquetField field;
+
+ public NestedColumnReader(boolean isUtcTimestamp, PageReadStore pages, ParquetField field) {
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.pages = pages;
+ this.field = field;
+ this.columnReaders = new HashMap<>();
+ }
+
+ @Override
+ public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
+ readData(field, readNumber, vector, false);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readData(
+ ParquetField field, int readNumber, ColumnVector vector, boolean inside) throws IOException {
+ if (field.getType() instanceof RowType) {
+ return readRow((ParquetGroupField) field, readNumber, vector, inside);
+ } else if (field.getType() instanceof MapType || field.getType() instanceof MultisetType) {
+ return readMap((ParquetGroupField) field, readNumber, vector, inside);
+ } else if (field.getType() instanceof ArrayType) {
+ return readArray((ParquetGroupField) field, readNumber, vector, inside);
+ } else {
+ return readPrimitive((ParquetPrimitiveField) field, readNumber, vector);
+ }
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readRow(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapRowColumnVector heapRowVector = (HeapRowColumnVector) vector;
+ LevelDelegation levelDelegation = null;
+ List<ParquetField> children = field.getChildren();
+ WritableColumnVector[] childrenVectors = heapRowVector.getFields();
+ WritableColumnVector[] finalChildrenVectors = new WritableColumnVector[childrenVectors.length];
+ for (int i = 0; i < children.size(); i++) {
+ ParquetField child = children.get(i);
+ if (child == null) {
+ // Hudi schema-evolution: the logical field is not present in the Parquet file. The slot
+ // vector was pre-populated with nulls by ParquetSplitReaderUtil#createWritableColumnVector
+ // (ROW branch); keep it as is and skip contributing to the level stream.
+ finalChildrenVectors[i] = childrenVectors[i];
+ continue;
+ }
+ Tuple2<LevelDelegation, WritableColumnVector> tuple =
+ readData(child, readNumber, childrenVectors[i], true);
+ levelDelegation = tuple.f0;
+ finalChildrenVectors[i] = tuple.f1;
+ }
+ if (levelDelegation == null) {
+ throw new FlinkRuntimeException(
+ String.format("Row field does not have any non-null children: %s.", field));
+ }
+
+ RowPosition rowPosition =
+ NestedPositionUtil.calculateRowOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If row was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ heapRowVector = new HeapRowColumnVector(rowPosition.getPositionsCount(), finalChildrenVectors);
+ } else {
+ heapRowVector.setFields(finalChildrenVectors);
+ }
+
+ if (rowPosition.getIsNull() != null) {
+ setFieldNullFlag(rowPosition.getIsNull(), heapRowVector);
+ }
+
+ // Hudi-specific: collapse a present row whose every child is null into a null row, so that a
+ // SQL value like `row(null, null)` round-trips to NULL on read. This was the behaviour of the
+ // legacy RowColumnReader (deleted alongside the Dremel rewire) and existing Hudi tables rely
+ // on it. Diverges from Flink 2.1, which would surface it as Row(null, null). Pinned by the
+ // integration test ITTestHoodieDataSource#testParquetNullChildColumnsRowTypes.
+ int rowCount = rowPosition.getPositionsCount();
+ for (int j = 0; j < rowCount; j++) {
+ if (heapRowVector.isNullAt(j)) {
+ continue;
+ }
+ boolean allChildrenNull = true;
+ for (WritableColumnVector child : finalChildrenVectors) {
+ if (!child.isNullAt(j)) {
+ allChildrenNull = false;
+ break;
+ }
+ }
+ if (allChildrenNull) {
+ heapRowVector.setNullAt(j);
+ }
+ }
+ return Tuple2.of(levelDelegation, heapRowVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readMap(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapMapColumnVector mapVector = (HeapMapColumnVector) vector;
+ mapVector.reset();
+ List<ParquetField> children = field.getChildren();
+ Preconditions.checkArgument(
+ children.size() == 2,
+ "Maps must have two type parameters, found %s",
+ children.size());
+ Tuple2<LevelDelegation, WritableColumnVector> keyTuple =
+ readData(children.get(0), readNumber, mapVector.getKeyColumnVector(), true);
+ Tuple2<LevelDelegation, WritableColumnVector> valueTuple =
+ readData(children.get(1), readNumber, mapVector.getValueColumnVector(), true);
+
+ LevelDelegation levelDelegation = keyTuple.f0;
+
+ CollectionPosition collectionPosition =
+ NestedPositionUtil.calculateCollectionOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If map was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ mapVector = new HeapMapColumnVector(collectionPosition.getValueCount(), keyTuple.f1, valueTuple.f1);
+ } else {
+ mapVector.setKeys(keyTuple.f1);
+ mapVector.setValues(valueTuple.f1);
+ }
+
+ if (collectionPosition.getIsNull() != null) {
+ setFieldNullFlag(collectionPosition.getIsNull(), mapVector);
+ }
+
+ mapVector.setLengths(collectionPosition.getLength());
+ mapVector.setOffsets(collectionPosition.getOffsets());
+
+ return Tuple2.of(levelDelegation, mapVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readArray(
+ ParquetGroupField field, int readNumber, ColumnVector vector, boolean inside)
+ throws IOException {
+ HeapArrayVector arrayVector = (HeapArrayVector) vector;
+ arrayVector.reset();
+ List<ParquetField> children = field.getChildren();
+ Preconditions.checkArgument(
+ children.size() == 1,
+ "Arrays must have a single type parameter, found %s",
+ children.size());
+ Tuple2<LevelDelegation, WritableColumnVector> tuple =
+ readData(children.get(0), readNumber, arrayVector.getChild(), true);
+
+ LevelDelegation levelDelegation = tuple.f0;
+ CollectionPosition collectionPosition =
+ NestedPositionUtil.calculateCollectionOffsets(
+ field,
+ levelDelegation.getDefinitionLevel(),
+ levelDelegation.getRepetitionLevel());
+
+ // If array was inside the structure, then we need to renew the vector to reset the
+ // capacity.
+ if (inside) {
+ arrayVector = new HeapArrayVector(collectionPosition.getValueCount(), tuple.f1);
+ } else {
+ arrayVector.setChild(tuple.f1);
+ }
+
+ if (collectionPosition.getIsNull() != null) {
+ setFieldNullFlag(collectionPosition.getIsNull(), arrayVector);
+ }
+ arrayVector.setLengths(collectionPosition.getLength());
+ arrayVector.setOffsets(collectionPosition.getOffsets());
+ return Tuple2.of(levelDelegation, arrayVector);
+ }
+
+ private Tuple2<LevelDelegation, WritableColumnVector> readPrimitive(
+ ParquetPrimitiveField field, int readNumber, ColumnVector vector) throws IOException {
+ ColumnDescriptor descriptor = field.getDescriptor();
+ NestedPrimitiveColumnReader reader = columnReaders.get(descriptor);
+ if (reader == null) {
+ reader =
+ new NestedPrimitiveColumnReader(
+ descriptor,
+ pages.getPageReader(descriptor),
+ isUtcTimestamp,
+ descriptor.getPrimitiveType(),
+ field.getType());
+ columnReaders.put(descriptor, reader);
+ }
+ WritableColumnVector writableColumnVector =
+ reader.readAndNewVector(readNumber, (WritableColumnVector) vector);
+ return Tuple2.of(reader.getLevelDelegation(), writableColumnVector);
+ }
+
+ private static void setFieldNullFlag(boolean[] nullFlags, AbstractHeapVector vector) {
+ for (int index = 0; index < vector.getLen() && index < nullFlags.length; index++) {
+ if (nullFlags[index]) {
+ vector.setNullAt(index);
+ }
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
new file mode 100644
index 000000000000..72809db1b2ce
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
@@ -0,0 +1,639 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.hudi.table.format.cow.utils.IntArrayList;
+import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
+import org.apache.hudi.table.format.cow.vector.position.LevelDelegation;
+
+import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBooleanVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapByteVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapDoubleVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapFloatVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapTimestampVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+import org.apache.flink.table.types.logical.LogicalType;
+
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.DataPageV1;
+import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.VALUES;
+
+/**
+ * Reader to read a single primitive leaf column that participates in a nested (Dremel) structure.
+ *
+ * <p>Vendored from Apache Flink 2.1 (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.reader.NestedPrimitiveColumnReader}). Only the package
+ * and the Hudi-local {@link ParquetDecimalVector} / {@link LevelDelegation} / {@link IntArrayList}
+ * imports are changed; the algorithm is untouched. The companion Hudi-specific {@code
+ * Int64TimestampColumnReader} / {@code FixedLenBytesColumnReader} behaviours stay at the leaf-
+ * reader creation boundary in {@code ParquetSplitReaderUtil}, not inside this class — keeping it
+ * a faithful copy of upstream.
+ */
+public class NestedPrimitiveColumnReader implements ColumnReader<WritableColumnVector> {
+ private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class);
+
+ private final IntArrayList repetitionLevelList = new IntArrayList(0);
+ private final IntArrayList definitionLevelList = new IntArrayList(0);
+
+ private final PageReader pageReader;
+ private final ColumnDescriptor descriptor;
+ private final Type type;
+ private final LogicalType logicalType;
+
+ /** The dictionary, if this column has dictionary encoding. */
+ private final ParquetDataColumnReader dictionary;
+
+ /** Maximum definition level for this column. */
+ private final int maxDefLevel;
+
+ private boolean isUtcTimestamp;
+
+ /** Total number of values read. */
+ private long valuesRead;
+
+ /**
+ * value that indicates the end of the current page. That is, if valuesRead ==
+ * endOfPageValueCount, we are at the end of the page.
+ */
+ private long endOfPageValueCount;
+
+ /** If true, the current page is dictionary encoded. */
+ private boolean isCurrentPageDictionaryEncoded;
+
+ private int definitionLevel;
+ private int repetitionLevel;
+
+ /** Repetition/Definition/Value readers. */
+ private IntIterator repetitionLevelColumn;
+
+ private IntIterator definitionLevelColumn;
+ private ParquetDataColumnReader dataColumn;
+
+ /** Total values in the current page. */
+ private int pageValueCount;
+
+ // flag to indicate if there is no data in parquet data page
+ private boolean eof = false;
+
+ private boolean isFirstRow = true;
+
+ private Object lastValue;
+
+ public NestedPrimitiveColumnReader(
+ ColumnDescriptor descriptor,
+ PageReader pageReader,
+ boolean isUtcTimestamp,
+ Type parquetType,
+ LogicalType logicalType)
+ throws IOException {
+ this.descriptor = descriptor;
+ this.type = parquetType;
+ this.pageReader = pageReader;
+ this.maxDefLevel = descriptor.getMaxDefinitionLevel();
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.logicalType = logicalType;
+
+ DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
+ if (dictionaryPage != null) {
+ try {
+ this.dictionary =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ parquetType.asPrimitiveType(),
+ dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } catch (IOException e) {
+ throw new IOException(
+ String.format("Could not decode the dictionary for %s", descriptor), e);
+ }
+ } else {
+ this.dictionary = null;
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+ }
+
+ // Not invoked directly; callers use readAndNewVector instead.
+ @Override
+ public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
+ throw new UnsupportedOperationException("This function should not be called.");
+ }
+
+ public WritableColumnVector readAndNewVector(int readNumber, WritableColumnVector vector)
+ throws IOException {
+ if (isFirstRow) {
+ if (!readValue()) {
+ return vector;
+ }
+ isFirstRow = false;
+ }
+
+ // index to set value.
+ int index = 0;
+ int valueIndex = 0;
+ List<Object> valueList = new ArrayList<>();
+
+ // repeated type need two loops to read data.
+ while (!eof && index < readNumber) {
+ do {
+ valueList.add(lastValue);
+ valueIndex++;
+ } while (readValue() && (repetitionLevel != 0));
+ index++;
+ }
+
+ return fillColumnVector(valueIndex, valueList);
+ }
+
+ public LevelDelegation getLevelDelegation() {
+ int[] repetition = repetitionLevelList.toArray();
+ int[] definition = definitionLevelList.toArray();
+ repetitionLevelList.clear();
+ definitionLevelList.clear();
+ repetitionLevelList.add(repetitionLevel);
+ definitionLevelList.add(definitionLevel);
+ return new LevelDelegation(repetition, definition);
+ }
+
+ private boolean readValue() throws IOException {
+ int left = readPageIfNeed();
+ if (left > 0) {
+ // get the values of repetition and definitionLevel
+ readAndSaveRepetitionAndDefinitionLevels();
+ // read the data if it isn't null
+ if (definitionLevel == maxDefLevel) {
+ if (isCurrentPageDictionaryEncoded) {
+ int dictionaryId = dataColumn.readValueDictionaryId();
+ lastValue = dictionaryDecodeValue(logicalType, dictionaryId);
+ } else {
+ lastValue = readPrimitiveTypedRow(logicalType);
+ }
+ } else {
+ lastValue = null;
+ }
+ return true;
+ } else {
+ eof = true;
+ return false;
+ }
+ }
+
+ private void readAndSaveRepetitionAndDefinitionLevels() {
+ // get the values of repetition and definitionLevel
+ repetitionLevel = repetitionLevelColumn.nextInt();
+ definitionLevel = definitionLevelColumn.nextInt();
+ valuesRead++;
+ repetitionLevelList.add(repetitionLevel);
+ definitionLevelList.add(definitionLevel);
+ }
+
+ private int readPageIfNeed() throws IOException {
+ // Compute the number of values we want to read in this page.
+ int leftInPage = (int) (endOfPageValueCount - valuesRead);
+ if (leftInPage == 0) {
+ // no data left in current page, load data from new page
+ readPage();
+ leftInPage = (int) (endOfPageValueCount - valuesRead);
+ }
+ return leftInPage;
+ }
+
+ private Object readPrimitiveTypedRow(LogicalType category) {
+ switch (category.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ return dataColumn.readBytes();
+ case BOOLEAN:
+ return dataColumn.readBoolean();
+ case TIME_WITHOUT_TIME_ZONE:
+ case DATE:
+ case INTEGER:
+ return dataColumn.readInteger();
+ case TINYINT:
+ return dataColumn.readTinyInt();
+ case SMALLINT:
+ return dataColumn.readSmallInt();
+ case BIGINT:
+ return dataColumn.readLong();
+ case FLOAT:
+ return dataColumn.readFloat();
+ case DOUBLE:
+ return dataColumn.readDouble();
+ case DECIMAL:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT32:
+ return dataColumn.readInteger();
+ case INT64:
+ return dataColumn.readLong();
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ return dataColumn.readBytes();
+ default:
+ throw new RuntimeException(
+ "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType());
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return dataColumn.readTimestamp();
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryValue) {
+ if (dictionaryValue == null) {
+ return null;
+ }
+
+ switch (category.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ return dictionary.readBytes(dictionaryValue);
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ case INTEGER:
+ return dictionary.readInteger(dictionaryValue);
+ case BOOLEAN:
+ return dictionary.readBoolean(dictionaryValue) ? 1 : 0;
+ case DOUBLE:
+ return dictionary.readDouble(dictionaryValue);
+ case FLOAT:
+ return dictionary.readFloat(dictionaryValue);
+ case TINYINT:
+ return dictionary.readTinyInt(dictionaryValue);
+ case SMALLINT:
+ return dictionary.readSmallInt(dictionaryValue);
+ case BIGINT:
+ return dictionary.readLong(dictionaryValue);
+ case DECIMAL:
+ switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
+ case INT32:
+ return dictionary.readInteger(dictionaryValue);
+ case INT64:
+ return dictionary.readLong(dictionaryValue);
+ case FIXED_LEN_BYTE_ARRAY:
+ case BINARY:
+ return dictionary.readBytes(dictionaryValue);
+ default:
+ throw new RuntimeException(
+ "Unsupported physical type for DECIMAL: " + descriptor.getPrimitiveType());
+ }
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return dictionary.readTimestamp(dictionaryValue);
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private WritableColumnVector fillColumnVector(int total, List valueList) {
+ switch (logicalType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ HeapBytesVector heapBytesVector = new HeapBytesVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ byte[] src = ((List<byte[]>) valueList).get(i);
+ if (src == null) {
+ heapBytesVector.setNullAt(i);
+ } else {
+ heapBytesVector.appendBytes(i, src, 0, src.length);
+ }
+ }
+ return heapBytesVector;
+ case BOOLEAN:
+ HeapBooleanVector heapBooleanVector = new HeapBooleanVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapBooleanVector.setNullAt(i);
+ } else {
+ heapBooleanVector.vector[i] = ((List<Boolean>) valueList).get(i);
+ }
+ }
+ return heapBooleanVector;
+ case TINYINT:
+ HeapByteVector heapByteVector = new HeapByteVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapByteVector.setNullAt(i);
+ } else {
+ heapByteVector.vector[i] = (byte) ((List<Integer>) valueList).get(i).intValue();
+ }
+ }
+ return heapByteVector;
+ case SMALLINT:
+ HeapShortVector heapShortVector = new HeapShortVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapShortVector.setNullAt(i);
+ } else {
+ heapShortVector.vector[i] = (short) ((List<Integer>) valueList).get(i).intValue();
+ }
+ }
+ return heapShortVector;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ HeapIntVector heapIntVector = new HeapIntVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapIntVector.setNullAt(i);
+ } else {
+ heapIntVector.vector[i] = ((List<Integer>) valueList).get(i);
+ }
+ }
+ return heapIntVector;
+ case FLOAT:
+ HeapFloatVector heapFloatVector = new HeapFloatVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapFloatVector.setNullAt(i);
+ } else {
+ heapFloatVector.vector[i] = ((List<Float>) valueList).get(i);
+ }
+ }
+ return heapFloatVector;
+ case BIGINT:
+ HeapLongVector heapLongVector = new HeapLongVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapLongVector.setNullAt(i);
+ } else {
+ heapLongVector.vector[i] = ((List<Long>) valueList).get(i);
+ }
+ }
+ return heapLongVector;
+ case DOUBLE:
+ HeapDoubleVector heapDoubleVector = new HeapDoubleVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapDoubleVector.setNullAt(i);
+ } else {
+ heapDoubleVector.vector[i] = ((List<Double>) valueList).get(i);
+ }
+ }
+ return heapDoubleVector;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HeapTimestampVector heapTimestampVector = new HeapTimestampVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ heapTimestampVector.setNullAt(i);
+ } else {
+ heapTimestampVector.setTimestamp(i, ((List<TimestampData>) valueList).get(i));
+ }
+ }
+ return heapTimestampVector;
+ case DECIMAL:
+ PrimitiveType.PrimitiveTypeName primitiveTypeName =
+ descriptor.getPrimitiveType().getPrimitiveTypeName();
+ switch (primitiveTypeName) {
+ case INT32:
+ HeapIntVector phiv = new HeapIntVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ phiv.setNullAt(i);
+ } else {
+ phiv.vector[i] = ((List<Integer>) valueList).get(i);
+ }
+ }
+ return new ParquetDecimalVector(phiv);
+ case INT64:
+ HeapLongVector phlv = new HeapLongVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ if (valueList.get(i) == null) {
+ phlv.setNullAt(i);
+ } else {
+ phlv.vector[i] = ((List<Long>) valueList).get(i);
+ }
+ }
+ return new ParquetDecimalVector(phlv);
+ default:
+ HeapBytesVector phbv = getHeapBytesVector(total, valueList);
+ return new ParquetDecimalVector(phbv);
+ }
+ default:
+ throw new RuntimeException("Unsupported type in the list: " + type);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static HeapBytesVector getHeapBytesVector(int total, List valueList) {
+ HeapBytesVector phbv = new HeapBytesVector(total);
+ for (int i = 0; i < valueList.size(); i++) {
+ byte[] src = ((List<byte[]>) valueList).get(i);
+ if (valueList.get(i) == null) {
+ phbv.setNullAt(i);
+ } else {
+ phbv.appendBytes(i, src, 0, src.length);
+ }
+ }
+ return phbv;
+ }
+
+ protected void readPage() {
+ DataPage page = pageReader.readPage();
+
+ if (page == null) {
+ return;
+ }
+
+ page.accept(
+ new DataPage.Visitor<Void>() {
+ @Override
+ public Void visit(DataPageV1 dataPageV1) {
+ readPageV1(dataPageV1);
+ return null;
+ }
+
+ @Override
+ public Void visit(DataPageV2 dataPageV2) {
+ readPageV2(dataPageV2);
+ return null;
+ }
+ });
+ }
+
+ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount)
+ throws IOException {
+ this.pageValueCount = valueCount;
+ this.endOfPageValueCount = valuesRead + pageValueCount;
+ if (dataEncoding.usesDictionary()) {
+ this.dataColumn = null;
+ if (dictionary == null) {
+ throw new IOException(
+ String.format(
+ "Could not read page in col %s because the dictionary was missing for encoding %s.",
+ descriptor, dataEncoding));
+ }
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getDictionaryBasedValuesReader(
+ descriptor, VALUES, dictionary.getDictionary()),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = true;
+ } else {
+ dataColumn =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type.asPrimitiveType(),
+ dataEncoding.getValuesReader(descriptor, VALUES),
+ isUtcTimestamp);
+ this.isCurrentPageDictionaryEncoded = false;
+ }
+
+ try {
+ dataColumn.initFromPage(pageValueCount, in);
+ } catch (IOException e) {
+ throw new IOException(String.format("Could not read page in col %s.", descriptor), e);
+ }
+ }
+
+ private void readPageV1(DataPageV1 page) {
+ ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
+ ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL);
+ this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
+ this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
+ try {
+ BytesInput bytes = page.getBytes();
+ LOG.debug("Page size {} bytes and {} records.", bytes.size(), pageValueCount);
+ ByteBufferInputStream in = bytes.toInputStream();
+ LOG.debug("Reading repetition levels at {}.", in.position());
+ rlReader.initFromPage(pageValueCount, in);
+ LOG.debug("Reading definition levels at {}.", in.position());
+ dlReader.initFromPage(pageValueCount, in);
+ LOG.debug("Reading data at {}.", in.position());
+ initDataReader(page.getValueEncoding(), in, page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read page %s in col %s.", page, descriptor), e);
+ }
+ }
+
+ private void readPageV2(DataPageV2 page) {
+ this.pageValueCount = page.getValueCount();
+ this.repetitionLevelColumn =
+ newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels());
+ this.definitionLevelColumn =
+ newRLEIterator(descriptor.getMaxDefinitionLevel(), page.getDefinitionLevels());
+ try {
+ LOG.debug(
+ "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount);
+ initDataReader(
+ page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount());
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read page %s in col %s.", page, descriptor), e);
+ }
+ }
+
+ private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
+ try {
+ if (maxLevel == 0) {
+ return new NullIntIterator();
+ }
+ return new RLEIntIterator(
+ new RunLengthBitPackingHybridDecoder(
+ BytesUtils.getWidthFromMaxInt(maxLevel),
+ new ByteArrayInputStream(bytes.toByteArray())));
+ } catch (IOException e) {
+ throw new ParquetDecodingException(
+ String.format("Could not read levels in page for col %s.", descriptor), e);
+ }
+ }
+
+ /** Utility interface to abstract over different way to read ints with different encodings. */
+ interface IntIterator {
+ int nextInt();
+ }
+
+ /** Reading int from {@link ValuesReader}. */
+ protected static final class ValuesReaderIntIterator implements IntIterator {
+ ValuesReader delegate;
+
+ public ValuesReaderIntIterator(ValuesReader delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int nextInt() {
+ return delegate.readInteger();
+ }
+ }
+
+ /** Reading int from {@link RunLengthBitPackingHybridDecoder}. */
+ protected static final class RLEIntIterator implements IntIterator {
+ RunLengthBitPackingHybridDecoder delegate;
+
+ public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int nextInt() {
+ try {
+ return delegate.readInt();
+ } catch (IOException e) {
+ throw new ParquetDecodingException(e);
+ }
+ }
+ }
+
+ /** Reading zero always. */
+ protected static final class NullIntIterator implements IntIterator {
+ @Override
+ public int nextInt() {
+ return 0;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index 3572b117a631..1826419db5d4 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -18,7 +18,9 @@
package org.apache.hudi.table.format.cow.vector.reader;
+import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil;
import org.apache.hudi.table.format.cow.vector.ParquetDecimalVector;
+import org.apache.hudi.table.format.cow.vector.type.ParquetField;
import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
import org.apache.flink.table.data.RowData;
@@ -28,6 +30,7 @@ import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch;
import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.util.FlinkRuntimeException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -39,6 +42,8 @@ import org.apache.parquet.filter2.predicate.FilterPredicate;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.io.ColumnIOFactory;
+import org.apache.parquet.io.MessageColumnIO;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
@@ -46,6 +51,7 @@ import org.apache.parquet.schema.Types;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -53,7 +59,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.stream.IntStream;
-import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader;
import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
@@ -77,6 +82,14 @@ public class ParquetColumnarRowSplitReader implements Closeable {
private final MessageType requestedSchema;
+ /**
+ * {@link ParquetField} tree per top-level requested column, used by
+ * {@link ParquetSplitReaderUtil#createColumnReader(boolean, LogicalType, Type, List,
+ * PageReadStore, ParquetField)} to drive the Dremel-style {@link NestedColumnReader} for
+ * nested types. Entries are {@code null} for primitive top-level fields. Built once per split.
+ */
+ private final List<ParquetField> requestedFields;
+
/**
* The total number of rows this RecordReader will eventually read. The sum of the rows of all
* the row groups.
@@ -158,6 +171,20 @@ public class ParquetColumnarRowSplitReader implements Closeable {
checkSchema();
+ // Build the ParquetField tree once per split (the Dremel-style nested reader reuses it across
+ // row groups). Only columns with nested logical type get a non-null entry — primitive columns
+ // still use Hudi's specialized ColumnReaders.
+ MessageColumnIO messageColumnIO = new ColumnIOFactory().getColumnIO(requestedSchema);
+ List<RowType.RowField> requestedRowFields = new ArrayList<>(requestedTypes.length);
+ List<String> requestedFieldNames = new ArrayList<>(requestedTypes.length);
+ for (int i = 0; i < requestedTypes.length; i++) {
+ String name = requestedSchema.getFieldName(i);
+ requestedRowFields.add(new RowType.RowField(name, requestedTypes[i]));
+ requestedFieldNames.add(name);
+ }
+ this.requestedFields = ParquetSplitReaderUtil.buildFieldsList(
+ requestedRowFields, requestedFieldNames, messageColumnIO);
+
this.writableVectors = createWritableVectors();
ColumnVector[] columnVectors = patchedVector(selectedFieldNames.length, createReadableVectors(), requestedIndices);
this.columnarBatch = generator.generate(columnVectors);
@@ -340,12 +367,13 @@ public class ParquetColumnarRowSplitReader implements Closeable {
List<ColumnDescriptor> columns = requestedSchema.getColumns();
columnReaders = new ColumnReader[types.size()];
for (int i = 0; i < types.size(); ++i) {
- columnReaders[i] = createColumnReader(
+ columnReaders[i] = ParquetSplitReaderUtil.createColumnReader(
utcTimestamp,
requestedTypes[i],
types.get(i),
columns,
- pages);
+ pages,
+ requestedFields.get(i));
}
totalCountLoadedSoFar += pages.getRowCount();
}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
index fdfe5d6fa3a3..1abc6ed56c0d 100644
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java
@@ -26,12 +26,16 @@ import org.apache.parquet.bytes.ByteBufferInputStream;
import org.apache.parquet.column.Dictionary;
import org.apache.parquet.column.values.ValuesReader;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.JULIAN_EPOCH_OFFSET_DAYS;
import static org.apache.flink.formats.parquet.vector.reader.TimestampColumnReader.MILLIS_IN_DAY;
@@ -252,21 +256,115 @@ public final class ParquetDataColumnReaderFactory {
}
}
+ /**
+ * Reader for Parquet INT64 timestamp values (MILLIS / MICROS / NANOS), i.e. the standard
+ * timestamp encoding defined by Parquet's
+ * {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and the legacy
+ * {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS} annotations.
+ * (The older INT96 encoding is marked deprecated by the Parquet format spec — see
+ * <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp">
+ * LogicalTypes.md</a> — but is still supported here via {@link TypesFromInt96PageReader} for
+ * backwards compatibility with files written by older Hive / Spark / Impala versions.)
+ *
+ * <p>Used by {@link NestedPrimitiveColumnReader} when a TIMESTAMP column sits inside a
+ * {@code Row}, {@code Array} or {@code Map}; the top-level path continues to use
+ * {@link Int64TimestampColumnReader} for batched-vector efficiency.
+ */
+ public static class TypesFromInt64PageReader extends DefaultParquetDataColumnReader {
+ private final boolean isUtcTimestamp;
+ private final ChronoUnit chronoUnit;
+
+ public TypesFromInt64PageReader(
+ ValuesReader realReader, boolean isUtcTimestamp, ChronoUnit chronoUnit) {
+ super(realReader);
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.chronoUnit = chronoUnit;
+ }
+
+ public TypesFromInt64PageReader(
+ Dictionary dict, boolean isUtcTimestamp, ChronoUnit chronoUnit) {
+ super(dict);
+ this.isUtcTimestamp = isUtcTimestamp;
+ this.chronoUnit = chronoUnit;
+ }
+
+ @Override
+ public TimestampData readTimestamp() {
+ return int64ToTimestamp(isUtcTimestamp, valuesReader.readLong(), chronoUnit);
+ }
+
+ @Override
+ public TimestampData readTimestamp(int id) {
+ return int64ToTimestamp(isUtcTimestamp, dict.decodeToLong(id), chronoUnit);
+ }
+ }
+
private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(
boolean isDictionary,
PrimitiveType parquetType,
Dictionary dictionary,
ValuesReader valuesReader,
boolean isUtcTimestamp) {
- if (parquetType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) {
+ PrimitiveType.PrimitiveTypeName typeName = parquetType.getPrimitiveTypeName();
+ if (typeName == PrimitiveType.PrimitiveTypeName.INT96) {
return isDictionary
? new TypesFromInt96PageReader(dictionary, isUtcTimestamp)
: new TypesFromInt96PageReader(valuesReader, isUtcTimestamp);
- } else {
- return isDictionary
- ? new DefaultParquetDataColumnReader(dictionary)
- : new DefaultParquetDataColumnReader(valuesReader);
}
+ if (typeName == PrimitiveType.PrimitiveTypeName.INT64) {
+ ChronoUnit unit = resolveInt64TimestampUnit(parquetType);
+ if (unit != null) {
+ return isDictionary
+ ? new TypesFromInt64PageReader(dictionary, isUtcTimestamp, unit)
+ : new TypesFromInt64PageReader(valuesReader, isUtcTimestamp, unit);
+ }
+ }
+ return isDictionary
+ ? new DefaultParquetDataColumnReader(dictionary)
+ : new DefaultParquetDataColumnReader(valuesReader);
+ }
+
+ /**
+ * Returns the {@link ChronoUnit} for a Parquet INT64 TIMESTAMP column, or {@code null} if the
+ * column is a plain INT64 (not a timestamp).
+ *
+ * <p>Supports both the modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation} and
+ * the legacy {@link OriginalType#TIMESTAMP_MILLIS} / {@link OriginalType#TIMESTAMP_MICROS}
+ * encodings.
+ */
+ private static ChronoUnit resolveInt64TimestampUnit(PrimitiveType parquetType) {
+ LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation();
+ if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) {
+ LogicalTypeAnnotation.TimeUnit unit =
+ ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation).getUnit();
+ switch (unit) {
+ case MILLIS:
+ return ChronoUnit.MILLIS;
+ case MICROS:
+ return ChronoUnit.MICROS;
+ case NANOS:
+ return ChronoUnit.NANOS;
+ default:
+ return null;
+ }
+ }
+ OriginalType originalType = parquetType.getOriginalType();
+ if (originalType == OriginalType.TIMESTAMP_MILLIS) {
+ return ChronoUnit.MILLIS;
+ }
+ if (originalType == OriginalType.TIMESTAMP_MICROS) {
+ return ChronoUnit.MICROS;
+ }
+ return null;
+ }
+
+ private static TimestampData int64ToTimestamp(
+ boolean isUtcTimestamp, long value, ChronoUnit unit) {
+ Instant instant = Instant.EPOCH.plus(value, unit);
+ if (isUtcTimestamp) {
+ return TimestampData.fromInstant(instant);
+ }
+ return TimestampData.fromTimestamp(Timestamp.from(instant));
}
public static ParquetDataColumnReader getDataColumnReaderByTypeOnDictionary(
@@ -281,10 +379,10 @@ public final class ParquetDataColumnReaderFactory {
}
private static TimestampData int96ToTimestamp(
- boolean utcTimestamp, long nanosOfDay, int julianDay) {
+ boolean isUtcTimestamp, long nanosOfDay, int julianDay) {
long millisecond = julianDayToMillis(julianDay) + (nanosOfDay / NANOS_PER_MILLISECOND);
- if (utcTimestamp) {
+ if (isUtcTimestamp) {
int nanoOfMillisecond = (int) (nanosOfDay % NANOS_PER_MILLISECOND);
return TimestampData.fromEpochMillis(millisecond, nanoOfMillisecond);
} else {
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
deleted file mode 100644
index 79b50487f13c..000000000000
--- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.table.format.cow.vector.reader;
-
-import org.apache.hudi.table.format.cow.vector.HeapRowColumnVector;
-
-import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
-import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Row {@link ColumnReader}.
- */
-public class RowColumnReader implements ColumnReader<WritableColumnVector> {
-
- private final List<ColumnReader> fieldReaders;
-
- public RowColumnReader(List<ColumnReader> fieldReaders) {
- this.fieldReaders = fieldReaders;
- }
-
- @Override
- public void readToVector(int readNumber, WritableColumnVector vector) throws IOException {
- HeapRowColumnVector rowColumnVector = (HeapRowColumnVector) vector;
- WritableColumnVector[] vectors = rowColumnVector.vectors;
- // row vector null array
- boolean[] isNulls = new boolean[readNumber];
- for (int i = 0; i < vectors.length; i++) {
- fieldReaders.get(i).readToVector(readNumber, vectors[i]);
-
- for (int j = 0; j < readNumber; j++) {
- if (i == 0) {
- isNulls[j] = vectors[i].isNullAt(j);
- } else {
- isNulls[j] = isNulls[j] && vectors[i].isNullAt(j);
- }
- if (i == vectors.length - 1 && isNulls[j]) {
- // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is
- // null
- rowColumnVector.setNullAt(j);
- }
- }
- }
- }
-}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
new file mode 100644
index 000000000000..0f5e00779a2f
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+
+/**
+ * Field that represent parquet's field type.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetField}).
+ */
+public abstract class ParquetField {
+ private final LogicalType type;
+ private final int repetitionLevel;
+ private final int definitionLevel;
+ private final boolean required;
+
+ public ParquetField(
+ LogicalType type, int repetitionLevel, int definitionLevel, boolean required) {
+ this.type = type;
+ this.repetitionLevel = repetitionLevel;
+ this.definitionLevel = definitionLevel;
+ this.required = required;
+ }
+
+ public LogicalType getType() {
+ return type;
+ }
+
+ public int getRepetitionLevel() {
+ return repetitionLevel;
+ }
+
+ public int getDefinitionLevel() {
+ return definitionLevel;
+ }
+
+ public boolean isRequired() {
+ return required;
+ }
+
+ @Override
+ public String toString() {
+ return "Field{"
+ + "type="
+ + type
+ + ", repetitionLevel="
+ + repetitionLevel
+ + ", definitionLevel="
+ + definitionLevel
+ + ", required="
+ + required
+ + '}';
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
new file mode 100644
index 000000000000..f91dcca965d6
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Field that represent parquet's Group Field.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetGroupField}) with a Hudi-specific extension:
+ * entries in the {@code children} list may be {@code null} to denote a Row child that is absent
+ * from the parquet file but present in the requested logical schema (schema evolution). This
+ * replaces Hudi's previous {@code EmptyColumnReader} branch for Row subtrees.
+ */
+public class ParquetGroupField extends ParquetField {
+
+ private final List<ParquetField> children;
+
+ public ParquetGroupField(
+ LogicalType type,
+ int repetitionLevel,
+ int definitionLevel,
+ boolean required,
+ List<ParquetField> children) {
+ super(type, repetitionLevel, definitionLevel, required);
+ // Use a plain unmodifiable list (not ImmutableList) so that null entries are allowed for
+ // schema-evolution missing children in ROW types.
+ this.children =
+ Collections.unmodifiableList(new ArrayList<>(requireNonNull(children, "children is null")));
+ }
+
+ /** Children of this group. Entries may be {@code null} for absent-in-file Row fields. */
+ public List<ParquetField> getChildren() {
+ return children;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
new file mode 100644
index 000000000000..f6af6f9ff479
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.parquet.column.ColumnDescriptor;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Field that represent parquet's primitive field.
+ *
+ * <p>Note: Vendored from Apache Flink (FLINK-35702, {@code
+ * org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField}).
+ */
+public class ParquetPrimitiveField extends ParquetField {
+
+ private final ColumnDescriptor descriptor;
+ private final int id;
+
+ public ParquetPrimitiveField(
+ LogicalType type, boolean required, ColumnDescriptor descriptor, int id) {
+ super(
+ type,
+ descriptor.getMaxRepetitionLevel(),
+ descriptor.getMaxDefinitionLevel(),
+ required);
+ this.descriptor = requireNonNull(descriptor, "descriptor is required");
+ this.id = id;
+ }
+
+ public ColumnDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ public int getId() {
+ return id;
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
new file mode 100644
index 000000000000..7cb62824e854
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.writable.WritableColumnVector;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+/**
+ * Tests for the Flink 2.1-compatible accessors added on {@link HeapArrayVector},
+ * {@link HeapMapColumnVector} and {@link HeapRowColumnVector} when vendoring Flink 2.1's
+ * nested-Parquet reader (FLINK-35702).
+ *
+ * <p>The accessors are wrappers over the existing public fields so legacy callers continue to
+ * work. These tests exist solely to pin down that wrapper contract — runtime correctness of the
+ * Dremel-style read path is exercised end-to-end by integration tests in
+ * {@code ITTestHoodieDataSource} (testParquetComplexTypes / testParquetComplexNestedRowTypes /
+ * testParquetArrayMapOfRowTypes / testParquetNullChildColumnsRowTypes).
+ */
+class TestHeapColumnVectorAccessors {
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapArrayVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapArrayVectorAccessorsReflectPublicFields() {
+ HeapIntVector child = new HeapIntVector(4);
+ HeapArrayVector vector = new HeapArrayVector(2, child);
+
+ long[] offsets = {0L, 2L};
+ long[] lengths = {2L, 2L};
+ HeapLongVector replacementChild = new HeapLongVector(4);
+
+ vector.setOffsets(offsets);
+ vector.setLengths(lengths);
+ vector.setChild(replacementChild);
+ vector.setSize(2);
+
+ assertArrayEquals(offsets, vector.getOffsets());
+ assertArrayEquals(lengths, vector.getLengths());
+ assertSame(replacementChild, vector.getChild());
+ assertEquals(2, vector.getSize());
+
+ // Backing public fields are kept in sync — preserves backward compatibility.
+ assertSame(offsets, vector.offsets);
+ assertSame(lengths, vector.lengths);
+ assertSame(replacementChild, vector.child);
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapMapColumnVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapMapColumnVectorConstructorInitializesOffsetsAndLengths() {
+ HeapIntVector keys = new HeapIntVector(4);
+ HeapIntVector values = new HeapIntVector(4);
+
+ HeapMapColumnVector vector = new HeapMapColumnVector(3, keys, values);
+
+ assertEquals(3, vector.getOffsets().length);
+ assertEquals(3, vector.getLengths().length);
+ }
+
+ @Test
+ void heapMapColumnVectorAccessorsReflectInternalState() {
+ HeapIntVector keys = new HeapIntVector(4);
+ HeapIntVector values = new HeapIntVector(4);
+ HeapMapColumnVector vector = new HeapMapColumnVector(2, keys, values);
+
+ long[] offsets = {0L, 2L};
+ long[] lengths = {2L, 2L};
+ HeapLongVector newKeys = new HeapLongVector(4);
+ HeapLongVector newValues = new HeapLongVector(4);
+
+ vector.setOffsets(offsets);
+ vector.setLengths(lengths);
+ vector.setKeys(newKeys);
+ vector.setValues(newValues);
+ vector.setSize(2);
+
+ assertArrayEquals(offsets, vector.getOffsets());
+ assertArrayEquals(lengths, vector.getLengths());
+ assertSame(newKeys, vector.getKeys());
+ assertSame(newValues, vector.getValues());
+ // The Flink-2.1-style ColumnVector accessors return the same underlying child.
+ assertSame(newKeys, vector.getKeyColumnVector());
+ assertSame(newValues, vector.getValueColumnVector());
+ assertEquals(2, vector.getSize());
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // HeapRowColumnVector
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void heapRowColumnVectorFieldsAccessorsReflectPublicVectors() {
+ HeapIntVector intField = new HeapIntVector(2);
+ HeapLongVector longField = new HeapLongVector(2);
+ HeapRowColumnVector vector = new HeapRowColumnVector(2, intField, longField);
+
+ WritableColumnVector[] originalFields = vector.getFields();
+ assertEquals(2, originalFields.length);
+ assertSame(intField, originalFields[0]);
+ assertSame(longField, originalFields[1]);
+ // Backing public field is kept in sync — preserves backward compatibility.
+ assertSame(originalFields, vector.vectors);
+
+ HeapIntVector replacement = new HeapIntVector(2);
+ WritableColumnVector[] replacementFields = {replacement, longField};
+ vector.setFields(replacementFields);
+
+ assertSame(replacementFields, vector.getFields());
+ assertSame(replacementFields, vector.vectors);
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
new file mode 100644
index 000000000000..02fe1e61ccb0
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector;
+
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.columnar.vector.BytesColumnVector;
+import org.apache.flink.table.data.columnar.vector.ColumnVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapBytesVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapIntVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapLongVector;
+import org.apache.flink.table.data.columnar.vector.heap.HeapShortVector;
+
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests for {@link ParquetDecimalVector}.
+ */
+public class TestParquetDecimalVector {
+
+ @Test
+ void testGetDecimalFromInt32Vector() {
+ // precision <= 9 => ParquetSchemaConverter.is32BitDecimal(precision) == true
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.vector[0] = 12345;
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 5, 2);
+
+ assertEquals(new BigDecimal("123.45"), decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromInt64Vector() {
+ // 9 < precision <= 18 => ParquetSchemaConverter.is64BitDecimal(precision) == true
+ HeapLongVector longVector = new HeapLongVector(1);
+ longVector.vector[0] = 1234567890123456L;
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 18, 4);
+
+ assertEquals(new BigDecimal("123456789012.3456"), decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromBytesVectorAtLargePrecision() {
+ // precision > 18 => BINARY / FIXED_LEN_BYTE_ARRAY path
+ BigDecimal original = new BigDecimal("12345678901234567890.1234567890");
+ byte[] unscaled = original.unscaledValue().toByteArray();
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ bytesVector.appendBytes(0, unscaled, 0, unscaled.length);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 30, 10);
+
+ assertEquals(original, decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalFromBytesVectorAtSmallPrecision() {
+ // A Parquet file can legally encode a small-precision decimal as BINARY. In that case the
+ // dispatch must fall through to the bytes branch rather than require an IntColumnVector.
+ BigDecimal original = new BigDecimal("123.45");
+ byte[] unscaled = original.unscaledValue().toByteArray();
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ bytesVector.appendBytes(0, unscaled, 0, unscaled.length);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+
+ DecimalData decoded = wrapped.getDecimal(0, 5, 2);
+
+ assertEquals(original, decoded.toBigDecimal());
+ }
+
+ @Test
+ void testGetDecimalThrowsOnUnsupportedVectorType() {
+ // A large-precision request must have a bytes-backed child; any other writable child is an
+ // illegal combination and must be surfaced via Preconditions.checkArgument.
+ ColumnVector unsupported = new HeapShortVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(unsupported);
+
+ assertThrows(IllegalArgumentException.class, () -> wrapped.getDecimal(0, 30, 10));
+ }
+
+ @Test
+ void testIsNullAtDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ intVector.vector[0] = 1;
+ intVector.setNullAt(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ assertFalse(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+
+ @Test
+ void testWritableIntRoundTrip() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.setInt(0, 42);
+
+ assertEquals(42, wrapped.getInt(0));
+ assertEquals(42, intVector.vector[0]);
+ }
+
+ @Test
+ void testWritableLongRoundTrip() {
+ HeapLongVector longVector = new HeapLongVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(longVector);
+
+ wrapped.setLong(0, 9876543210L);
+
+ assertEquals(9876543210L, wrapped.getLong(0));
+ assertEquals(9876543210L, longVector.vector[0]);
+ }
+
+ @Test
+ void testWritableBytesRoundTrip() {
+ HeapBytesVector bytesVector = new HeapBytesVector(1);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(bytesVector);
+ byte[] payload = new byte[] {0x01, 0x02, 0x03};
+
+ wrapped.appendBytes(0, payload, 0, payload.length);
+
+ BytesColumnVector.Bytes out = wrapped.getBytes(0);
+ assertEquals(payload.length, out.len);
+ assertEquals(0x01, out.data[out.offset]);
+ assertEquals(0x02, out.data[out.offset + 1]);
+ assertEquals(0x03, out.data[out.offset + 2]);
+ }
+
+ @Test
+ void testResetDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.setNullAt(0);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+ assertTrue(wrapped.isNullAt(0));
+
+ wrapped.reset();
+
+ assertFalse(wrapped.isNullAt(0));
+ }
+
+ @Test
+ void testFillWithNullsDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.fillWithNulls();
+
+ assertTrue(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+
+ @Test
+ void testSetNullAtDelegatesToChild() {
+ HeapIntVector intVector = new HeapIntVector(2);
+ ParquetDecimalVector wrapped = new ParquetDecimalVector(intVector);
+
+ wrapped.setNullAt(0);
+ wrapped.setNulls(1, 1);
+
+ assertTrue(wrapped.isNullAt(0));
+ assertTrue(wrapped.isNullAt(1));
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
new file mode 100644
index 000000000000..9d6607d03feb
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.reader;
+
+import org.apache.flink.table.data.TimestampData;
+
+import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Tests for the {@link ParquetDataColumnReaderFactory} INT64 timestamp dispatch added when
+ * vendoring Flink 2.1's nested-Parquet reader (FLINK-35702).
+ *
+ * <p>The factory is exercised end-to-end by integration tests through
+ * {@link NestedPrimitiveColumnReader}; this unit test focuses on the small, deterministic piece
+ * that was added by this PR — selecting the right {@code ParquetDataColumnReader} for each
+ * supported INT64 TIMESTAMP encoding (modern {@link LogicalTypeAnnotation.TimestampLogicalTypeAnnotation}
+ * MILLIS / MICROS / NANOS plus the legacy {@link OriginalType} encodings) and decoding values
+ * using both the values-reader and dictionary code paths.
+ */
+class TestParquetDataColumnReaderFactory {
+
+ // -----------------------------------------------------------------------------------------------
+ // Type dispatch
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void valuesReaderDispatchInt96TimestampUsesInt96Reader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT96).named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt96PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64WithoutAnnotationUsesDefaultReader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT64).named("plainLong");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampMicrosLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(false, LogicalTypeAnnotation.TimeUnit.MICROS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64TimestampNanosLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64LegacyTimestampMillisOriginalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(OriginalType.TIMESTAMP_MILLIS)
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt64LegacyTimestampMicrosOriginalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(OriginalType.TIMESTAMP_MICROS)
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ @Test
+ void valuesReaderDispatchInt32DoesNotUseTimestampReader() {
+ PrimitiveType type = Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("i");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(type, new StubValuesReader(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.DefaultParquetDataColumnReader.class, reader);
+ }
+
+ @Test
+ void dictionaryReaderDispatchInt64TimestampMillisLogicalUsesInt64Reader() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ type, new StubDictionary(), true);
+ assertInstanceOf(ParquetDataColumnReaderFactory.TypesFromInt64PageReader.class, reader);
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // INT64 → TimestampData decoding (per ChronoUnit, both UTC and local-time-zone branches)
+ // -----------------------------------------------------------------------------------------------
+
+ @Test
+ void int64ReaderReadsTimestampMillisFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ long epochMillis = 1_700_000_000_123L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochMillis), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochMillis, ts.getMillisecond());
+ assertEquals(0, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampMicrosFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS))
+ .named("ts");
+ long epochMicros = 1_700_000_000_123_456L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochMicros), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochMicros / 1_000L, ts.getMillisecond());
+ // 456 microseconds remain → 456_000 nanoseconds within the millisecond
+ assertEquals(456_000, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampNanosFromValuesReaderInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("ts");
+ long epochNanos = 1_700_000_000_123_456_789L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByType(
+ type, new StubValuesReader(epochNanos), true);
+
+ TimestampData ts = reader.readTimestamp();
+ assertNotNull(ts);
+ assertEquals(epochNanos / 1_000_000L, ts.getMillisecond());
+ assertEquals(456_789, ts.getNanoOfMillisecond());
+ }
+
+ @Test
+ void int64ReaderReadsTimestampMillisFromDictionaryInUtc() {
+ PrimitiveType type =
+ Types.required(PrimitiveType.PrimitiveTypeName.INT64)
+ .as(LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+ .named("ts");
+ long epochMillis = 1_700_000_000_456L;
+ ParquetDataColumnReader reader =
+ ParquetDataColumnReaderFactory.getDataColumnReaderByTypeOnDictionary(
+ type, new StubDictionary(epochMillis), true);
+
+ TimestampData ts = reader.readTimestamp(0);
+ assertNotNull(ts);
+ assertEquals(epochMillis, ts.getMillisecond());
+ }
+
+ // -----------------------------------------------------------------------------------------------
+ // Stubs (only the methods exercised by the dispatch + decoding tests above)
+ // -----------------------------------------------------------------------------------------------
+
+ /** Minimal {@link ValuesReader} returning a fixed long; other methods throw. */
+ private static final class StubValuesReader extends ValuesReader {
+ private final long fixedLong;
+
+ StubValuesReader() {
+ this(0L);
+ }
+
+ StubValuesReader(long fixedLong) {
+ this.fixedLong = fixedLong;
+ }
+
+ @Override
+ public long readLong() {
+ return fixedLong;
+ }
+
+ @Override
+ public void skip() {
+ // unused
+ }
+ }
+
+ /** Minimal {@link Dictionary} returning a fixed long for any id; other methods throw. */
+ private static final class StubDictionary extends Dictionary {
+ private final long fixedLong;
+
+ StubDictionary() {
+ this(0L);
+ }
+
+ StubDictionary(long fixedLong) {
+ super(null);
+ this.fixedLong = fixedLong;
+ }
+
+ @Override
+ public Binary decodeToBinary(int id) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long decodeToLong(int id) {
+ return fixedLong;
+ }
+
+ @Override
+ public int getMaxId() {
+ return 0;
+ }
+ }
+}
diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
new file mode 100644
index 000000000000..a071ef937611
--- /dev/null
+++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.format.cow.vector.type;
+
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
+
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Tests for {@link ParquetGroupField}.
+ */
+public class TestParquetGroupField {
+
+ @Test
+ void testChildrenWithAllNonNullEntriesAreRetained() {
+ ParquetField c0 = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ ParquetField c1 = new ParquetPrimitiveField(new VarCharType(), true, descriptor(), 1);
+ List<ParquetField> children = Arrays.asList(c0, c1);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children);
+
+ assertEquals(2, group.getChildren().size());
+ assertSame(c0, group.getChildren().get(0));
+ assertSame(c1, group.getChildren().get(1));
+ }
+
+ @Test
+ void testChildrenMayContainNullForSchemaEvolution() {
+ // A ROW field present in the requested Flink schema but absent from the Parquet file is
+ // represented by a null slot in `children`. The group must allow this (Hudi-specific
+ // extension over Flink's ImmutableList-backed equivalent).
+ ParquetField present = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ List<ParquetField> children = Arrays.asList(present, null);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, children);
+
+ assertEquals(2, group.getChildren().size());
+ assertNotNull(group.getChildren().get(0));
+ assertNull(group.getChildren().get(1));
+ }
+
+ @Test
+ void testChildrenListIsUnmodifiable() {
+ ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 0, 1, true, Collections.singletonList(child));
+
+ assertThrows(UnsupportedOperationException.class, () -> group.getChildren().add(null));
+ assertThrows(UnsupportedOperationException.class, () -> group.getChildren().remove(0));
+ }
+
+ @Test
+ void testChildrenListIsDefensivelyCopied() {
+ // Mutations to the caller-supplied list must not be visible through the group.
+ ParquetField child = new ParquetPrimitiveField(new IntType(), true, descriptor(), 0);
+ List<ParquetField> mutable = new ArrayList<>();
+ mutable.add(child);
+
+ ParquetGroupField group = new ParquetGroupField(rowType(), 0, 1, true, mutable);
+ mutable.add(null);
+
+ assertEquals(1, group.getChildren().size());
+ }
+
+ @Test
+ void testNullChildrenListThrows() {
+ assertThrows(
+ NullPointerException.class,
+ () -> new ParquetGroupField(rowType(), 0, 1, true, null));
+ }
+
+ @Test
+ void testEmptyChildrenListIsAllowed() {
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 0, 1, true, Collections.emptyList());
+
+ assertEquals(0, group.getChildren().size());
+ }
+
+ @Test
+ void testFieldMetadataIsExposed() {
+ ParquetGroupField group =
+ new ParquetGroupField(rowType(), 2, 5, false, Collections.emptyList());
+
+ assertEquals(2, group.getRepetitionLevel());
+ assertEquals(5, group.getDefinitionLevel());
+ assertFalse(group.isRequired());
+ }
+
+ private static LogicalType rowType() {
+ return RowType.of(new IntType());
+ }
+
+ private static ColumnDescriptor descriptor() {
+ PrimitiveType primitive = Types.required(PrimitiveTypeName.INT32).named("f");
+ return new ColumnDescriptor(new String[] {"f"}, primitive, 0, 0);
+ }
+}
(hudi) 06/12: chore: Add release information for Apache Hudi 1.2.0 (#18831)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 9c3137ae3c4589d131271e90bd5a1ddd9c32149b
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Mon May 25 19:29:07 2026 -0700
chore: Add release information for Apache Hudi 1.2.0 (#18831)
(cherry picked from commit 62ad88a3dc938d5362082fed4135af8cfe528bdb)
---
doap_HUDI.rdf | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/doap_HUDI.rdf b/doap_HUDI.rdf
index e76272529d32..921decaeec72 100644
--- a/doap_HUDI.rdf
+++ b/doap_HUDI.rdf
@@ -229,6 +229,13 @@
<revision>1.1.1</revision>
</Version>
</release>
+ <release>
+ <Version>
+ <name>Apache Hudi 1.2.0</name>
+ <created>2026-05-23</created>
+ <revision>1.2.0</revision>
+ </Version>
+ </release>
<repository>
<GitRepository>
<location rdf:resource="https://github.com/apache/hudi.git"/>
(hudi) 02/12: chore: add spark4.1 and flink2.1 profile entries to RC bundle validation (#18796)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 88b4e85809b115dc714c4bab0f654d0b252702be
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed May 20 20:14:13 2026 -0700
chore: add spark4.1 and flink2.1 profile entries to RC bundle validation (#18796)
(cherry picked from commit a687786833a616cb0f646d96726d4ab231f64ecc)
---
.github/workflows/release_candidate_validation.yml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml
index cde2891ee3a2..ff4605c02475 100644
--- a/.github/workflows/release_candidate_validation.yml
+++ b/.github/workflows/release_candidate_validation.yml
@@ -77,6 +77,10 @@ jobs:
flinkProfile: 'flink1.20'
sparkProfile: 'spark4.0'
sparkRuntime: 'spark4.0.0'
+ - scalaProfile: 'scala-2.13'
+ flinkProfile: 'flink1.20'
+ sparkProfile: 'spark4.1'
+ sparkRuntime: 'spark4.1.1'
steps:
- uses: actions/checkout@v5
- name: Set up JDK 17
@@ -108,6 +112,10 @@ jobs:
flinkProfile: 'flink2.0'
sparkProfile: 'spark3.5'
sparkRuntime: 'spark3.5.1'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink2.1'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.1'
steps:
- uses: actions/checkout@v5
- name: Set up JDK 11