You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2025/08/06 23:22:54 UTC
(hudi) branch master updated: [HUDI-9556] Migrate trino-hudi plugin to hudi repo (#13493)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4e987fe936d1 [HUDI-9556] Migrate trino-hudi plugin to hudi repo (#13493)
4e987fe936d1 is described below
commit 4e987fe936d1ad9089906d46bb57dc193d89fc75
Author: voonhous <vo...@gmail.com>
AuthorDate: Thu Aug 7 07:22:48 2025 +0800
[HUDI-9556] Migrate trino-hudi plugin to hudi repo (#13493)
Co-authored-by: Y Ethan Guo <et...@gmail.com>
Co-authored-by: Sagar Sumit <sa...@gmail.com>
---
.github/workflows/bot.yml | 22 +
.../modernizer/violations-production-code-only.xml | 86 ++
hudi-trino-plugin/.mvn/modernizer/violations.xml | 365 ++++++
hudi-trino-plugin/pom.xml | 460 +++++++
.../io/trino/plugin/hudi/ForHudiSplitManager.java | 29 +
.../io/trino/plugin/hudi/ForHudiSplitSource.java | 29 +
.../plugin/hudi/HudiBaseFileOnlyPageSource.java | 126 ++
.../main/java/io/trino/plugin/hudi/HudiConfig.java | 410 ++++++
.../java/io/trino/plugin/hudi/HudiConnector.java | 158 +++
.../io/trino/plugin/hudi/HudiConnectorFactory.java | 118 ++
.../java/io/trino/plugin/hudi/HudiErrorCode.java | 51 +
.../java/io/trino/plugin/hudi/HudiFileStatus.java | 26 +
.../java/io/trino/plugin/hudi/HudiMetadata.java | 498 ++++++++
.../io/trino/plugin/hudi/HudiMetadataFactory.java | 58 +
.../main/java/io/trino/plugin/hudi/HudiModule.java | 117 ++
.../java/io/trino/plugin/hudi/HudiPageSource.java | 143 +++
.../trino/plugin/hudi/HudiPageSourceProvider.java | 451 +++++++
.../main/java/io/trino/plugin/hudi/HudiPlugin.java | 28 +
.../java/io/trino/plugin/hudi/HudiPredicates.java | 68 +
.../trino/plugin/hudi/HudiSessionProperties.java | 395 ++++++
.../main/java/io/trino/plugin/hudi/HudiSplit.java | 135 ++
.../io/trino/plugin/hudi/HudiSplitManager.java | 140 +++
.../java/io/trino/plugin/hudi/HudiSplitSource.java | 239 ++++
.../java/io/trino/plugin/hudi/HudiTableHandle.java | 201 +++
.../java/io/trino/plugin/hudi/HudiTableInfo.java | 28 +
.../java/io/trino/plugin/hudi/HudiTableName.java | 65 +
.../io/trino/plugin/hudi/HudiTableProperties.java | 70 ++
.../trino/plugin/hudi/HudiTransactionManager.java | 80 ++
.../main/java/io/trino/plugin/hudi/HudiUtil.java | 286 +++++
.../main/java/io/trino/plugin/hudi/TableType.java | 20 +
.../java/io/trino/plugin/hudi/TimelineTable.java | 92 ++
.../io/trino/plugin/hudi/file/HudiBaseFile.java | 113 ++
.../java/io/trino/plugin/hudi/file/HudiFile.java | 29 +
.../io/trino/plugin/hudi/file/HudiLogFile.java | 113 ++
.../plugin/hudi/io/HudiTrinoFileReaderFactory.java | 73 ++
.../trino/plugin/hudi/io/HudiTrinoIOFactory.java | 63 +
.../hudi/io/InlineSeekableDataInputStream.java | 51 +
.../hudi/io/TrinoSeekableDataInputStream.java | 45 +
.../hudi/partition/HiveHudiPartitionInfo.java | 113 ++
.../plugin/hudi/partition/HudiPartitionInfo.java | 27 +
.../hudi/partition/HudiPartitionInfoLoader.java | 90 ++
.../plugin/hudi/query/HudiDirectoryLister.java | 29 +
.../hudi/query/HudiSnapshotDirectoryLister.java | 123 ++
.../hudi/query/index/HudiBaseIndexSupport.java | 66 +
.../query/index/HudiColumnStatsIndexSupport.java | 319 +++++
.../plugin/hudi/query/index/HudiIndexSupport.java | 27 +
.../hudi/query/index/HudiNoOpIndexSupport.java | 40 +
.../index/HudiPartitionStatsIndexSupport.java | 146 +++
.../query/index/HudiRecordLevelIndexSupport.java | 327 +++++
.../query/index/HudiSecondaryIndexSupport.java | 232 ++++
.../hudi/query/index/IndexSupportFactory.java | 148 +++
.../plugin/hudi/reader/HudiTrinoReaderContext.java | 227 ++++
.../trino/plugin/hudi/reader/HudiTrinoRecord.java | 183 +++
.../hudi/split/HudiBackgroundSplitLoader.java | 145 +++
.../trino/plugin/hudi/split/HudiSplitFactory.java | 128 ++
.../plugin/hudi/split/HudiSplitWeightProvider.java | 26 +
.../hudi/split/SizeBasedSplitWeightProvider.java | 47 +
.../plugin/hudi/stats/ForHudiTableStatistics.java | 29 +
.../plugin/hudi/stats/HudiTableStatistics.java | 23 +
.../plugin/hudi/stats/TableMetadataReader.java | 114 ++
.../plugin/hudi/stats/TableStatisticsReader.java | 118 ++
.../hudi/storage/HudiTrinoInlineStorage.java | 242 ++++
.../plugin/hudi/storage/HudiTrinoStorage.java | 287 +++++
.../hudi/storage/TrinoStorageConfiguration.java | 84 ++
.../trino/plugin/hudi/util/HudiAvroSerializer.java | 504 ++++++++
.../trino/plugin/hudi/util/HudiTableTypeUtils.java | 44 +
.../plugin/hudi/util/SynthesizedColumnHandler.java | 320 +++++
.../hudi/util/SynthesizedColumnStrategy.java | 25 +
.../trino/plugin/hudi/util/TupleDomainUtils.java | 162 +++
.../plugin/hudi/BaseHudiConnectorSmokeTest.java | 59 +
.../java/io/trino/plugin/hudi/HudiQueryRunner.java | 170 +++
.../java/io/trino/plugin/hudi/SessionBuilder.java | 137 +++
.../java/io/trino/plugin/hudi/TestHudiConfig.java | 122 ++
.../plugin/hudi/TestHudiConnectorFactory.java | 70 ++
.../TestHudiConnectorParquetColumnNamesTest.java | 31 +
.../trino/plugin/hudi/TestHudiConnectorTest.java | 90 ++
.../hudi/TestHudiMinioConnectorSmokeTest.java | 41 +
.../io/trino/plugin/hudi/TestHudiPageSource.java | 28 +
.../hudi/TestHudiPageSourceProviderTest.java | 242 ++++
.../java/io/trino/plugin/hudi/TestHudiPlugin.java | 38 +
.../plugin/hudi/TestHudiSessionProperties.java | 39 +
.../trino/plugin/hudi/TestHudiSharedMetastore.java | 178 +++
.../io/trino/plugin/hudi/TestHudiSmokeTest.java | 1301 ++++++++++++++++++++
.../io/trino/plugin/hudi/TestHudiSystemTables.java | 61 +
.../plugin/hudi/TestingHudiConnectorFactory.java | 64 +
.../io/trino/plugin/hudi/TestingHudiPlugin.java | 43 +
.../index/HudiRecordLevelIndexSupportTest.java | 484 ++++++++
.../hudi/query/index/TestingColumnHandle.java | 60 +
.../plugin/hudi/split/TestHudiSplitFactory.java | 202 +++
.../plugin/hudi/testing/HudiTableUnzipper.java | 125 ++
.../plugin/hudi/testing/HudiTablesInitializer.java | 23 +
.../trino/plugin/hudi/testing/HudiTestUtils.java | 29 +
.../testing/ResourceHudiTablesInitializer.java | 753 +++++++++++
.../hudi/testing/TpchHudiTablesInitializer.java | 400 ++++++
.../trino/plugin/hudi/testing/TypeInfoHelper.java | 129 ++
...TestTupleDomainUtilsExtendedNullFilterTest.java | 194 +++
.../plugin/hudi/util/TestTupleDomainUtilsTest.java | 269 ++++
.../hudi_comprehensive_types_v6_mor.md | 164 +++
.../hudi_comprehensive_types_v6_mor.zip | Bin 0 -> 257256 bytes
.../hudi_comprehensive_types_v8_mor.md | 168 +++
.../hudi_comprehensive_types_v8_mor.zip | Bin 0 -> 378428 bytes
.../hudi-testing-data/hudi_cow_pt_tbl.zip | Bin 0 -> 11750 bytes
.../hudi_custom_keygen_pt_v8_mor.md | 64 +
.../hudi_custom_keygen_pt_v8_mor.zip | Bin 0 -> 226852 bytes
.../hudi-testing-data/hudi_multi_fg_pt_v6_mor.md | 51 +
.../hudi-testing-data/hudi_multi_fg_pt_v6_mor.zip | Bin 0 -> 158908 bytes
.../hudi-testing-data/hudi_multi_fg_pt_v8_mor.md | 56 +
.../hudi-testing-data/hudi_multi_fg_pt_v8_mor.zip | Bin 0 -> 260372 bytes
.../hudi-testing-data/hudi_multi_pt_v8_mor.md | 56 +
.../hudi-testing-data/hudi_multi_pt_v8_mor.zip | Bin 0 -> 493901 bytes
.../hudi-testing-data/hudi_non_part_cow.md | 47 +
.../hudi-testing-data/hudi_non_part_cow.zip | Bin 0 -> 5980 bytes
.../hudi-testing-data/hudi_stock_ticks_cow.zip | Bin 0 -> 45382 bytes
.../hudi-testing-data/hudi_stock_ticks_mor.zip | Bin 0 -> 62909 bytes
...tamp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.md | 63 +
...amp_keygen_pt_epoch_to_yyyy_mm_dd_hh_v8_mor.zip | Bin 0 -> 327289 bytes
...amp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.md | 64 +
...mp_keygen_pt_scalar_to_yyyy_mm_dd_hh_v8_mor.zip | Bin 0 -> 327962 bytes
.../hudi-testing-data/hudi_trips_cow_v8.md | 34 +
.../hudi-testing-data/hudi_trips_cow_v8.zip | Bin 0 -> 3860131 bytes
.../hudi-testing-data/hudi_v6_composite_key.zip | Bin 0 -> 284603 bytes
.../hudi-testing-data/stock_ticks_cow.zip | Bin 0 -> 18891 bytes
.../hudi-testing-data/stock_ticks_mor.zip | Bin 0 -> 24925 bytes
.../src/test/resources/long_timestamp.parquet | Bin 0 -> 598 bytes
scripts/release/validate_source_copyright.sh | 4 +-
style/checkstyle-suppressions.xml | 2 +
126 files changed, 16029 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 033e4b9c69b3..9d7209acf32a 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1245,3 +1245,25 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
mvn test -Punit-tests -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS
+
+ test-hudi-trino-plugin:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ - name: Set up JDK 23
+ uses: actions/setup-java@v3
+ with:
+ # Note: We are not caching here again, as we want to use the .m2 repository populated by
+ # the previous step
+ java-version: '23'
+ distribution: 'temurin'
+ architecture: x64
+ - name: Build hudi-trino-plugin with JDK 23
+ working-directory: ./hudi-trino-plugin
+ run:
+ mvn clean install -DskipTests
+ - name: Test hudi-trino-plugin with JDK 23
+ working-directory: ./hudi-trino-plugin
+ run:
+ mvn test
diff --git a/hudi-trino-plugin/.mvn/modernizer/violations-production-code-only.xml b/hudi-trino-plugin/.mvn/modernizer/violations-production-code-only.xml
new file mode 100644
index 000000000000..4653d63fafc2
--- /dev/null
+++ b/hudi-trino-plugin/.mvn/modernizer/violations-production-code-only.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<modernizer>
+ <violation>
+ <name>java/util/concurrent/ThreadPoolExecutor."<init>":(IIJLjava/util/concurrent/TimeUnit;Ljava/util/concurrent/BlockingQueue;)V</name>
+ <version>1.1</version>
+ <comment>Use constructor that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/ThreadPoolExecutor."<init>":(IIJLjava/util/concurrent/TimeUnit;Ljava/util/concurrent/BlockingQueue;Ljava/util/concurrent/RejectedExecutionHandler;)V</name>
+ <version>1.1</version>
+ <comment>Use constructor that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/ScheduledThreadPoolExecutor."<init>":(I)V</name>
+ <version>1.1</version>
+ <comment>Use constructor that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/ScheduledThreadPoolExecutor."<init>":(ILjava/util/concurrent/RejectedExecutionHandler;)V</name>
+ <version>1.1</version>
+ <comment>Use constructor that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newFixedThreadPool:(I)Ljava/util/concurrent/ExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newWorkStealingPool:()Ljava/util/concurrent/ExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newWorkStealingPool:(I)Ljava/util/concurrent/ExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newSingleThreadExecutor:()Ljava/util/concurrent/ExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newCachedThreadPool:()Ljava/util/concurrent/ExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newSingleThreadScheduledExecutor:()Ljava/util/concurrent/ScheduledExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/concurrent/Executors.newScheduledThreadPool:(I)Ljava/util/concurrent/ScheduledExecutorService;</name>
+ <version>1.1</version>
+ <comment>Use factory method that takes ThreadFactory and name the threads</comment>
+ </violation>
+</modernizer>
diff --git a/hudi-trino-plugin/.mvn/modernizer/violations.xml b/hudi-trino-plugin/.mvn/modernizer/violations.xml
new file mode 100644
index 000000000000..eae8ea39b591
--- /dev/null
+++ b/hudi-trino-plugin/.mvn/modernizer/violations.xml
@@ -0,0 +1,365 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<modernizer>
+ <violation>
+ <name>java/lang/Class.newInstance:()Ljava/lang/Object;</name>
+ <version>1.1</version>
+ <comment>Prefer Class.getConstructor().newInstance()</comment>
+ </violation>
+
+ <violation>
+ <name>java/lang/String."<init>":([B)V</name>
+ <version>1.1</version>
+ <comment>Prefer new String(byte[], Charset)</comment>
+ </violation>
+
+ <violation>
+ <name>java/lang/String.getBytes:()[B</name>
+ <version>1.1</version>
+ <comment>Prefer String.getBytes(Charset)</comment>
+ </violation>
+
+ <violation>
+ <name>java/lang/String.toString:()Ljava/lang/String;</name>
+ <version>1.1</version>
+ <comment>Call to toString() is redundant</comment>
+ </violation>
+
+ <violation>
+ <!-- File.getPath() is the canonical way to convert File to its String representation suitable for passing to new File() -->
+ <name>java/io/File.toString:()Ljava/lang/String;</name>
+ <version>1.1</version>
+ <comment>Prefer File.getPath()</comment>
+ </violation>
+
+ <violation>
+ <name>java/lang/Thread$Builder.factory:()Ljava/util/concurrent/ThreadFactory;</name>
+ <version>1.1</version>
+ <comment>Use io.airlift.concurrent.Threads's thread factories, as the set thread context class loader</comment>
+ </violation>
+ <violation>
+ <name>java/lang/Thread$Builder$OfPlatform.factory:()Ljava/util/concurrent/ThreadFactory;</name>
+ <version>1.1</version>
+ <comment>Use io.airlift.concurrent.Threads's thread factories, as the set thread context class loader</comment>
+ </violation>
+ <violation>
+ <name>java/lang/Thread$Builder$OfVirtual.factory:()Ljava/util/concurrent/ThreadFactory;</name>
+ <version>1.1</version>
+ <comment>Use io.airlift.concurrent.Threads's thread factories, as the set thread context class loader</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/common/primitives/Ints.checkedCast:(J)I</name>
+ <version>1.8</version>
+ <comment>Prefer Math.toIntExact(long)</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/common/collect/ImmutableMap$Builder.build:()Lcom/google/common/collect/ImmutableMap;</name>
+ <version>1.8</version>
+ <comment>Use buildOrThrow() instead, as it makes it clear that it will throw on duplicated values</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableTable$Builder.build:()Lcom/google/common/collect/ImmutableTable;</name>
+ <version>1.8</version>
+ <comment>Use buildOrThrow() instead, as it makes it clear that it will throw on duplicated values</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/common/collect/ImmutableBiMap$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableList$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableMap$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableMultimap$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableMultiset$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableSet$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableSortedMap$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use orderedBy() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableSortedSet$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use orderedBy() static factory method instead</comment>
+ </violation>
+ <violation>
+ <name>com/google/common/collect/ImmutableTable$Builder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use builder() static factory method instead</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/common/cache/CacheBuilder.build:()Lcom/google/common/cache/Cache;</name>
+ <version>1.8</version>
+ <comment>Guava Cache has concurrency issues around invalidation and ongoing loads. Use EvictableCacheBuilder or SafeCaches to build caches.
+ See https://github.com/trinodb/trino/issues/10512 for more information and see https://github.com/trinodb/trino/issues/10512#issuecomment-1016221168
+ for why Caffeine does not solve the problem.</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/common/cache/CacheBuilder.build:(Lcom/google/common/cache/CacheLoader;)Lcom/google/common/cache/LoadingCache;</name>
+ <version>1.8</version>
+ <comment>Guava LoadingCache has concurrency issues around invalidation and ongoing loads. Use EvictableCacheBuilder or SafeCaches to build caches.
+ See https://github.com/trinodb/trino/issues/10512 for more information and see https://github.com/trinodb/trino/issues/10512#issuecomment-1016221168
+ for why Caffeine does not solve the problem.</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/Assert.assertEquals:(Ljava/lang/Iterable;Ljava/lang/Iterable;)V</name>
+ <version>1.8</version>
+ <comment>Use AssertJ or QueryAssertions due to TestNG #543</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/Assert.assertEquals:(Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/String;)V</name>
+ <version>1.8</version>
+ <comment>Use AssertJ or QueryAssertions due to TestNG #543</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/Assert.assertThrows:(Lorg/testng/Assert$ThrowingRunnable;)V</name>
+ <version>1.8</version>
+ <comment>Use AssertJ's assertThatThrownBy, see https://github.com/trinodb/trino/issues/5320 for rationale</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/Assert.assertThrows:(Ljava/lang/Class;Lorg/testng/Assert$ThrowingRunnable;)V</name>
+ <version>1.8</version>
+ <comment>Use AssertJ's assertThatThrownBy, see https://github.com/trinodb/trino/issues/5320 for rationale</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/Table.getStorageDescriptor:()Lcom/amazonaws/services/glue/model/StorageDescriptor;</name>
+ <version>1.1</version>
+ <comment>Storage descriptor is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getStorageDescriptor</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/Table.getTableType:()Ljava/lang/String;</name>
+ <version>1.1</version>
+ <comment>Table type is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getTableType</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/Column.getParameters:()Ljava/util/Map;</name>
+ <version>1.1</version>
+ <comment>Column parameters map is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getColumnParameters</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/Table.getParameters:()Ljava/util/Map;</name>
+ <version>1.1</version>
+ <comment>Table parameters map is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getTableParameters</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/Partition.getParameters:()Ljava/util/Map;</name>
+ <version>1.1</version>
+ <comment>Partition parameters map is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getPartitionParameters</comment>
+ </violation>
+
+ <violation>
+ <name>com/amazonaws/services/glue/model/SerDeInfo.getParameters:()Ljava/util/Map;</name>
+ <version>1.1</version>
+ <comment>SerDeInfo parameters map is nullable in Glue model, which is too easy to forget about. Prefer GlueToTrinoConverter.getSerDeInfoParameters</comment>
+ </violation>
+
+ <violation>
+ <name>org/apache/hadoop/fs/FileSystem.close:()V</name>
+ <version>1.1</version>
+ <comment>Hadoop FileSystem instances are shared and should not be closed</comment>
+ </violation>
+
+ <violation>
+ <name>java/util/TimeZone.getTimeZone:(Ljava/lang/String;)Ljava/util/TimeZone;</name>
+ <version>1.8</version>
+ <comment>Avoid TimeZone.getTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(..)) instead, or TimeZone.getTimeZone(..., false).</comment>
+ </violation>
+
+ <violation>
+ <name>org/joda/time/DateTimeZone.toTimeZone:()Ljava/util/TimeZone;</name>
+ <version>1.8</version>
+ <comment>Avoid DateTimeZone.toTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(dtz.getId())) instead.</comment>
+ </violation>
+
+ <violation>
+ <name>com/esri/core/geometry/ogc/OGCGeometry.equals:(Lcom/esri/core/geometry/ogc/OGCGeometry;)Z</name>
+ <version>1.6</version>
+ <comment>Prefer OGCGeometry.Equals(OGCGeometry)</comment>
+ </violation>
+
+ <violation>
+ <name>com/esri/core/geometry/ogc/OGCGeometry.equals:(Ljava/lang/Object;)Z</name>
+ <version>1.6</version>
+ <comment>Prefer OGCGeometry.Equals(OGCGeometry)</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize."<init>":(DLio/airlift/units/DataSize$Unit;)V</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.of(long, DataSize.Unit)</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.succinctDataSize:(DLio/airlift/units/DataSize$Unit;)Lio/airlift/units/DataSize;</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.of(long, DataSize.Unit).succinct() -- Note that succinct conversion only affects toString() results</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.getValue:()D</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.toBytes() and Unit.inBytes() for conversion</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.getValue:(Lio/airlift/units/DataSize$Unit;)D</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.toBytes() and Unit.inBytes() for conversion</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.roundTo:(Lio/airlift/units/DataSize$Unit;)J</name>
+ <version>1.8</version>
+ <comment>Method is deprecated for removal</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.convertTo:(Lio/airlift/units/DataSize$Unit;)Lio/airlift/units/DataSize;</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.to(DataSize.Unit)</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/units/DataSize.convertToMostSuccinctDataSize:()Lio/airlift/units/DataSize;</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.units.DataSize.succinct()</comment>
+ </violation>
+
+ <violation>
+ <name>io/airlift/testing/Closeables.closeQuietly:([Ljava/io/Closeable;)V</name>
+ <version>1.0</version>
+ <comment>Use Closeables.closeAll() or Closer.</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/inject/util/Modules.combine:(Ljava/lang/Iterable;)Lcom/google/inject/Module;</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.configuration.ConfigurationAwareModule.combine</comment>
+ </violation>
+
+ <violation>
+ <name>com/google/inject/util/Modules.combine:([Lcom/google/inject/Module;)Lcom/google/inject/Module;</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.configuration.ConfigurationAwareModule.combine</comment>
+ </violation>
+
+ <violation>
+ <name>io/jsonwebtoken/Jwts.builder:()Lio/jsonwebtoken/JwtBuilder;</name>
+ <version>1.8</version>
+ <comment>Use io.trino.server.security.jwt.JwtsUtil or equivalent</comment>
+ </violation>
+
+ <violation>
+ <name>io/jsonwebtoken/Jwts.parserBuilder:()Lio/jsonwebtoken/JwtParserBuilder;</name>
+ <version>1.8</version>
+ <comment>Use io.trino.server.security.jwt.JwtsUtil or equivalent</comment>
+ </violation>
+
+ <violation>
+ <name>org/openjdk/jol/info/ClassLayout.instanceSize:()J</name>
+ <version>1.8</version>
+ <comment>Use io.airlift.slice.SizeOf.instanceSize</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/annotations/BeforeTest</name>
+ <version>1.8</version>
+ <comment>Prefer org.testng.annotations.BeforeClass</comment>
+ </violation>
+
+ <violation>
+ <name>org/testng/annotations/AfterTest</name>
+ <version>1.8</version>
+ <comment>Prefer org.testng.annotations.AfterClass</comment>
+ </violation>
+
+ <violation>
+ <name>com/fasterxml/jackson/core/JsonFactory."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use io.trino.plugin.base.util.JsonUtils.jsonFactory()</comment>
+ </violation>
+
+ <violation>
+ <name>com/fasterxml/jackson/core/JsonFactoryBuilder."<init>":()V</name>
+ <version>1.8</version>
+ <comment>Use io.trino.plugin.base.util.JsonUtils.jsonFactoryBuilder() instead</comment>
+ </violation>
+
+ <violation>
+ <name>software/amazon/awssdk/services/glue/model/Table.tableType:()Ljava/lang/String;</name>
+ <version>1.8</version>
+ <comment>Table type is nullable in Glue model, which is too easy to forget about. Prefer GlueConverter.getTableTypeNullable</comment>
+ </violation>
+
+ <violation>
+ <name>org/assertj/core/util/Files.newTemporaryFile:()Ljava/io/File;</name>
+ <version>1.1</version>
+ <comment>Use @TempDir instead</comment>
+ </violation>
+ <violation>
+ <name>org/assertj/core/util/Files.newTemporaryFolder:()Ljava/io/File;</name>
+ <version>1.1</version>
+ <comment>Use @TempDir instead</comment>
+ </violation>
+ <violation>
+ <name>org/assertj/core/util/Files.temporaryFolder:()Ljava/io/File;</name>
+ <version>1.1</version>
+ <comment>Use @TempDir instead</comment>
+ </violation>
+ <violation>
+ <name>org/assertj/core/util/Files.temporaryFolderPath:()Ljava/lang/String;</name>
+ <version>1.1</version>
+ <comment>Use @TempDir instead</comment>
+ </violation>
+</modernizer>
diff --git a/hudi-trino-plugin/pom.xml b/hudi-trino-plugin/pom.xml
new file mode 100644
index 000000000000..d819b3093e23
--- /dev/null
+++ b/hudi-trino-plugin/pom.xml
@@ -0,0 +1,460 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-root</artifactId>
+ <version>472</version>
+ <!-- Stop looking for the parent on local file system -->
+ <relativePath />
+ </parent>
+
+ <artifactId>trino-hudi</artifactId>
+ <packaging>trino-plugin</packaging>
+ <description>Trino - Hudi connector</description>
+
+ <properties>
+ <air.compiler.fail-warnings>true</air.compiler.fail-warnings>
+ <dep.hudi.version>1.0.2</dep.hudi.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <!--Used to test execution in task executor after de-serializing-->
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo</artifactId>
+ <version>4.0.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_annotations</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>bootstrap</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>concurrent</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>configuration</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>log</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>units</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-filesystem</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-filesystem-manager</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-hive</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-memory-context</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-metastore</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-parquet</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-plugin-toolkit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>jakarta.validation</groupId>
+ <artifactId>jakarta.validation-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-common</artifactId>
+ <version>${dep.hudi.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-io</artifactId>
+ <version>${dep.hudi.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-column</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.weakref</groupId>
+ <artifactId>jmxutils</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>slice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-api-incubator</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.opentelemetry</groupId>
+ <artifactId>opentelemetry-context</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-spi</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.openjdk.jol</groupId>
+ <artifactId>jol-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>log-manager</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-hive-formats</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>junit-extensions</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>testing</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-hdfs</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-hive</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-main</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-main</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-parser</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-spi</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-testing</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-testing-containers</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-testing-services</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino</groupId>
+ <artifactId>trino-tpch</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino.hadoop</groupId>
+ <artifactId>hadoop-apache</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.trino.tpch</groupId>
+ <artifactId>tpch</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-client-common</artifactId>
+ <version>${dep.hudi.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-hadoop-common</artifactId>
+ <version>${dep.hudi.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-java-client</artifactId>
+ <version>${dep.hudi.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20250107</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.basepom.maven</groupId>
+ <artifactId>duplicate-finder-maven-plugin</artifactId>
+ <configuration>
+ <ignoredResourcePatterns>
+ <!-- org.apache.hudi:hudi-client-common and org.apache.hudi:hudi-java-client log4j.properties duplicates -->
+ <ignoredResourcePattern>log4j.properties</ignoredResourcePattern>
+ <ignoredResourcePattern>log4j-surefire.properties</ignoredResourcePattern>
+ </ignoredResourcePatterns>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>8</source>
+ <target>8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java
new file mode 100644
index 000000000000..0f1987594a1a
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitManager.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@BindingAnnotation
+public @interface ForHudiSplitManager {}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java
new file mode 100644
index 000000000000..801b1e030940
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/ForHudiSplitSource.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@BindingAnnotation
+public @interface ForHudiSplitSource {}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java
new file mode 100644
index 000000000000..1180638a7cc4
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiBaseFileOnlyPageSource.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.SynthesizedColumnHandler;
+import io.trino.spi.Page;
+import io.trino.spi.block.Block;
+import io.trino.spi.connector.ConnectorPageSource;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * This page source is for reading data columns in the parquet format.
+ * This page source also avoids costly avro IndexRecord serialization.
+ */
+public class HudiBaseFileOnlyPageSource
+ implements ConnectorPageSource
+{
+ private final ConnectorPageSource dataPageSource;
+ private final List<HiveColumnHandle> allOutputColumns;
+ private final SynthesizedColumnHandler synthesizedColumnHandler;
+ // Maps output channel to physical source channel, or -1 if synthesized
+ private final int[] physicalSourceChannelMap;
+
+ public HudiBaseFileOnlyPageSource(
+ ConnectorPageSource dataPageSource,
+ List<HiveColumnHandle> allOutputColumns,
+ // Columns provided by dataPageSource
+ List<HiveColumnHandle> dataColumns,
+ // Handler to manage synthesized/virtual in Hudi tables such as partition columns and metadata, i.e. file size (not hudi metadata)
+ SynthesizedColumnHandler synthesizedColumnHandler)
+ {
+ this.dataPageSource = requireNonNull(dataPageSource, "dataPageSource is null");
+ this.allOutputColumns = ImmutableList.copyOf(requireNonNull(allOutputColumns, "allOutputColumns is null"));
+ this.synthesizedColumnHandler = requireNonNull(synthesizedColumnHandler, "synthesizedColumnHandler is null");
+
+ // Create a mapping from the channel index in the output page to the channel index in the physicalDataPageSource's page
+ this.physicalSourceChannelMap = new int[allOutputColumns.size()];
+ Map<String, Integer> physicalColumnNameToChannel = new HashMap<>();
+ for (int i = 0; i < dataColumns.size(); i++) {
+ physicalColumnNameToChannel.put(dataColumns.get(i).getName().toLowerCase(Locale.ENGLISH), i);
+ }
+
+ for (int i = 0; i < allOutputColumns.size(); i++) {
+ this.physicalSourceChannelMap[i] = physicalColumnNameToChannel.getOrDefault(allOutputColumns.get(i).getName().toLowerCase(Locale.ENGLISH), -1);
+ }
+ }
+
+ @Override
+ public long getCompletedBytes()
+ {
+ return dataPageSource.getCompletedBytes();
+ }
+
+ @Override
+ public long getReadTimeNanos()
+ {
+ return dataPageSource.getReadTimeNanos();
+ }
+
+ @Override
+ public boolean isFinished()
+ {
+ return dataPageSource.isFinished();
+ }
+
+ @Override
+ public Page getNextPage()
+ {
+ Page physicalSourcePage = dataPageSource.getNextPage();
+ if (physicalSourcePage == null) {
+ return null;
+ }
+
+ int positionCount = physicalSourcePage.getPositionCount();
+ if (positionCount == 0 && synthesizedColumnHandler.getSynthesizedColumnCount() == 0) {
+ // If only physical columns and page is empty
+ return physicalSourcePage;
+ }
+
+ Block[] outputBlocks = new Block[allOutputColumns.size()];
+ for (int i = 0; i < allOutputColumns.size(); i++) {
+ HiveColumnHandle outputColumn = allOutputColumns.get(i);
+ if (physicalSourceChannelMap[i] != -1) {
+ outputBlocks[i] = physicalSourcePage.getBlock(physicalSourceChannelMap[i]);
+ }
+ else {
+ // Column is synthesized
+ outputBlocks[i] = synthesizedColumnHandler.createRleSynthesizedBlock(outputColumn, positionCount);
+ }
+ }
+ return new Page(outputBlocks);
+ }
+
+ @Override
+ public long getMemoryUsage()
+ {
+ return dataPageSource.getMemoryUsage();
+ }
+
+ @Override
+ public void close()
+ throws IOException
+ {
+ dataPageSource.close();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java
new file mode 100644
index 000000000000..5138caea39c8
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConfig.java
@@ -0,0 +1,410 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import io.airlift.configuration.Config;
+import io.airlift.configuration.ConfigDescription;
+import io.airlift.configuration.DefunctConfig;
+import io.airlift.units.DataSize;
+import io.airlift.units.Duration;
+import jakarta.validation.constraints.DecimalMax;
+import jakarta.validation.constraints.DecimalMin;
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotNull;
+
+import java.util.List;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.airlift.units.DataSize.Unit.MEGABYTE;
+import static java.util.Locale.ENGLISH;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+@DefunctConfig({
+ "hudi.min-partition-batch-size",
+ "hudi.max-partition-batch-size"
+})
+public class HudiConfig
+{
+ private List<String> columnsToHide = ImmutableList.of();
+ private boolean tableStatisticsEnabled = true;
+ private int tableStatisticsExecutorParallelism = 4;
+ private boolean metadataEnabled;
+ private boolean shouldUseParquetColumnNames = true;
+ private boolean shouldUseParquetColumnIndex;
+ private boolean sizeBasedSplitWeightsEnabled = true;
+ private DataSize standardSplitWeightSize = DataSize.of(128, MEGABYTE);
+ private double minimumAssignedSplitWeight = 0.05;
+ private DataSize targetSplitSize = DataSize.of(128, MEGABYTE);
+ private int maxSplitsPerSecond = Integer.MAX_VALUE;
+ private int maxOutstandingSplits = 1000;
+ private int splitLoaderParallelism = 4;
+ private int splitGeneratorParallelism = 4;
+ private long perTransactionMetastoreCacheMaximumSize = 2000;
+ private boolean queryPartitionFilterRequired;
+ private boolean ignoreAbsentPartitions;
+ private Duration dynamicFilteringWaitTimeout = new Duration(1, SECONDS);
+
+ // Internal configuration for debugging and testing
+ private boolean isRecordLevelIndexEnabled = true;
+ private boolean isSecondaryIndexEnabled = true;
+ private boolean isColumnStatsIndexEnabled = true;
+ private boolean isPartitionStatsIndexEnabled = true;
+ private Duration columnStatsWaitTimeout = new Duration(1, SECONDS);
+ private Duration recordIndexWaitTimeout = new Duration(2, SECONDS);
+ private Duration secondaryIndexWaitTimeout = new Duration(2, SECONDS);
+
+ public List<String> getColumnsToHide()
+ {
+ return columnsToHide;
+ }
+
+ @Config("hudi.columns-to-hide")
+ @ConfigDescription("List of column names that will be hidden from the query output. " +
+ "It can be used to hide Hudi meta fields. By default, no fields are hidden.")
+ public HudiConfig setColumnsToHide(List<String> columnsToHide)
+ {
+ this.columnsToHide = columnsToHide.stream()
+ .map(s -> s.toLowerCase(ENGLISH))
+ .collect(toImmutableList());
+ return this;
+ }
+
+ @Config("hudi.table-statistics-enabled")
+ @ConfigDescription("Enable table statistics for query planning.")
+ public HudiConfig setTableStatisticsEnabled(boolean tableStatisticsEnabled)
+ {
+ this.tableStatisticsEnabled = tableStatisticsEnabled;
+ return this;
+ }
+
+ public boolean isTableStatisticsEnabled()
+ {
+ return this.tableStatisticsEnabled;
+ }
+
+ @Min(1)
+ public int getTableStatisticsExecutorParallelism()
+ {
+ return tableStatisticsExecutorParallelism;
+ }
+
+ @Config("hudi.table-statistics-executor-parallelism")
+ @ConfigDescription("Number of threads to asynchronously generate table statistics.")
+ public HudiConfig setTableStatisticsExecutorParallelism(int parallelism)
+ {
+ this.tableStatisticsExecutorParallelism = parallelism;
+ return this;
+ }
+
+ @Config("hudi.metadata-enabled")
+ @ConfigDescription("Fetch the list of file names and sizes from Hudi metadata table rather than storage.")
+ public HudiConfig setMetadataEnabled(boolean metadataEnabled)
+ {
+ this.metadataEnabled = metadataEnabled;
+ return this;
+ }
+
+ public boolean isMetadataEnabled()
+ {
+ return this.metadataEnabled;
+ }
+
+ @Config("hudi.parquet.use-column-names")
+ @ConfigDescription("Access Parquet columns using names from the file. If disabled, then columns are accessed using index."
+ + "Only applicable to Parquet file format.")
+ public HudiConfig setUseParquetColumnNames(boolean shouldUseParquetColumnNames)
+ {
+ this.shouldUseParquetColumnNames = shouldUseParquetColumnNames;
+ return this;
+ }
+
+ public boolean getUseParquetColumnNames()
+ {
+ return this.shouldUseParquetColumnNames;
+ }
+
+ @Config("hudi.parquet.use-column-index")
+ @ConfigDescription("Enable using Parquet column indexes")
+ public HudiConfig setUseParquetColumnIndex(boolean shouldUseParquetColumnIndex)
+ {
+ this.shouldUseParquetColumnIndex = shouldUseParquetColumnIndex;
+ return this;
+ }
+
+ public boolean isUseParquetColumnIndex()
+ {
+ return this.shouldUseParquetColumnIndex;
+ }
+
+ @Config("hudi.size-based-split-weights-enabled")
+ @ConfigDescription("Unlike uniform splitting, size-based splitting ensures that each batch of splits has enough data to process. " +
+ "By default, it is enabled to improve performance.")
+ public HudiConfig setSizeBasedSplitWeightsEnabled(boolean sizeBasedSplitWeightsEnabled)
+ {
+ this.sizeBasedSplitWeightsEnabled = sizeBasedSplitWeightsEnabled;
+ return this;
+ }
+
+ public boolean isSizeBasedSplitWeightsEnabled()
+ {
+ return sizeBasedSplitWeightsEnabled;
+ }
+
+ @Config("hudi.standard-split-weight-size")
+ @ConfigDescription("The split size corresponding to the standard weight (1.0) "
+ + "when size based split weights are enabled.")
+ public HudiConfig setStandardSplitWeightSize(DataSize standardSplitWeightSize)
+ {
+ this.standardSplitWeightSize = standardSplitWeightSize;
+ return this;
+ }
+
+ @NotNull
+ public DataSize getStandardSplitWeightSize()
+ {
+ return standardSplitWeightSize;
+ }
+
+ @Config("hudi.minimum-assigned-split-weight")
+ @ConfigDescription("Minimum weight that a split can be assigned when size based split weights are enabled.")
+ public HudiConfig setMinimumAssignedSplitWeight(double minimumAssignedSplitWeight)
+ {
+ this.minimumAssignedSplitWeight = minimumAssignedSplitWeight;
+ return this;
+ }
+
+ @DecimalMax("1")
+ @DecimalMin(value = "0", inclusive = false)
+ public double getMinimumAssignedSplitWeight()
+ {
+ return minimumAssignedSplitWeight;
+ }
+
+ @Config("hudi.target-split-size")
+ @ConfigDescription("The target split size")
+ public HudiConfig setTargetSplitSize(DataSize targetSplitSize)
+ {
+ this.targetSplitSize = targetSplitSize;
+ return this;
+ }
+
+ @NotNull
+ public DataSize getTargetSplitSize()
+ {
+ return targetSplitSize;
+ }
+
+ @Min(1)
+ public int getMaxSplitsPerSecond()
+ {
+ return maxSplitsPerSecond;
+ }
+
+ @Config("hudi.max-splits-per-second")
+ @ConfigDescription("Rate at which splits are enqueued for processing. The queue will throttle if this rate limit is breached.")
+ public HudiConfig setMaxSplitsPerSecond(int maxSplitsPerSecond)
+ {
+ this.maxSplitsPerSecond = maxSplitsPerSecond;
+ return this;
+ }
+
+ @Min(1)
+ public int getMaxOutstandingSplits()
+ {
+ return maxOutstandingSplits;
+ }
+
+ @Config("hudi.max-outstanding-splits")
+ @ConfigDescription("Maximum outstanding splits in a batch enqueued for processing.")
+ public HudiConfig setMaxOutstandingSplits(int maxOutstandingSplits)
+ {
+ this.maxOutstandingSplits = maxOutstandingSplits;
+ return this;
+ }
+
+ @Min(1)
+ public int getSplitGeneratorParallelism()
+ {
+ return splitGeneratorParallelism;
+ }
+
+ @Config("hudi.split-generator-parallelism")
+ @ConfigDescription("Number of threads to generate splits from partitions.")
+ public HudiConfig setSplitGeneratorParallelism(int splitGeneratorParallelism)
+ {
+ this.splitGeneratorParallelism = splitGeneratorParallelism;
+ return this;
+ }
+
+ @Min(1)
+ public int getSplitLoaderParallelism()
+ {
+ return splitLoaderParallelism;
+ }
+
+ @Config("hudi.split-loader-parallelism")
+ @ConfigDescription("Number of threads to run background split loader. A single background split loader is needed per query.")
+ public HudiConfig setSplitLoaderParallelism(int splitLoaderParallelism)
+ {
+ this.splitLoaderParallelism = splitLoaderParallelism;
+ return this;
+ }
+
+ @Min(1)
+ public long getPerTransactionMetastoreCacheMaximumSize()
+ {
+ return perTransactionMetastoreCacheMaximumSize;
+ }
+
+ @Config("hudi.per-transaction-metastore-cache-maximum-size")
+ public HudiConfig setPerTransactionMetastoreCacheMaximumSize(long perTransactionMetastoreCacheMaximumSize)
+ {
+ this.perTransactionMetastoreCacheMaximumSize = perTransactionMetastoreCacheMaximumSize;
+ return this;
+ }
+
+ @Config("hudi.query-partition-filter-required")
+ @ConfigDescription("Require a filter on at least one partition column")
+ public HudiConfig setQueryPartitionFilterRequired(boolean queryPartitionFilterRequired)
+ {
+ this.queryPartitionFilterRequired = queryPartitionFilterRequired;
+ return this;
+ }
+
+ public boolean isQueryPartitionFilterRequired()
+ {
+ return queryPartitionFilterRequired;
+ }
+
+ @Config("hudi.ignore-absent-partitions")
+ public HudiConfig setIgnoreAbsentPartitions(boolean ignoreAbsentPartitions)
+ {
+ this.ignoreAbsentPartitions = ignoreAbsentPartitions;
+ return this;
+ }
+
+ public boolean isIgnoreAbsentPartitions()
+ {
+ return ignoreAbsentPartitions;
+ }
+
+ @Config("hudi.index.record-level-index-enabled")
+ @ConfigDescription("Internal configuration to control whether record level index is enabled for debugging/testing.")
+ public HudiConfig setRecordLevelIndexEnabled(boolean isRecordLevelIndexEnabled)
+ {
+ this.isRecordLevelIndexEnabled = isRecordLevelIndexEnabled;
+ return this;
+ }
+
+ public boolean isRecordLevelIndexEnabled()
+ {
+ return isRecordLevelIndexEnabled;
+ }
+
+ @Config("hudi.index.secondary-index-enabled")
+ @ConfigDescription("Internal configuration to control whether secondary index is enabled for debugging/testing.")
+ public HudiConfig setSecondaryIndexEnabled(boolean isSecondaryIndexEnabled)
+ {
+ this.isSecondaryIndexEnabled = isSecondaryIndexEnabled;
+ return this;
+ }
+
+ public boolean isSecondaryIndexEnabled()
+ {
+ return isSecondaryIndexEnabled;
+ }
+
+ @Config("hudi.index.column-stats-index-enabled")
+ @ConfigDescription("Internal configuration to control whether column stats index is enabled for debugging/testing.")
+ public HudiConfig setColumnStatsIndexEnabled(boolean isColumnStatsIndexEnabled)
+ {
+ this.isColumnStatsIndexEnabled = isColumnStatsIndexEnabled;
+ return this;
+ }
+
+ public boolean isColumnStatsIndexEnabled()
+ {
+ return isColumnStatsIndexEnabled;
+ }
+
+ @Config("hudi.index.partition-stats-index-enabled")
+ @ConfigDescription("Internal configuration to control whether partition stats index is enabled for debugging/testing.")
+ public HudiConfig setPartitionStatsIndexEnabled(boolean isPartitionStatsIndexEnabled)
+ {
+ this.isPartitionStatsIndexEnabled = isPartitionStatsIndexEnabled;
+ return this;
+ }
+
+ public boolean isPartitionStatsIndexEnabled()
+ {
+ return isPartitionStatsIndexEnabled;
+ }
+
+ @Config("hudi.dynamic-filtering.wait-timeout")
+ @ConfigDescription("Maximum timeout to wait for dynamic filtering, e.g. 1000ms, 20s, 2m, 1h")
+ public HudiConfig setDynamicFilteringWaitTimeout(Duration dynamicFilteringWaitTimeout)
+ {
+ this.dynamicFilteringWaitTimeout = dynamicFilteringWaitTimeout;
+ return this;
+ }
+
+ @NotNull
+ public Duration getDynamicFilteringWaitTimeout()
+ {
+ return dynamicFilteringWaitTimeout;
+ }
+
+ @Config("hudi.index.column-stats.wait-timeout")
+ @ConfigDescription("Maximum timeout to wait for loading column stats, e.g. 1000ms, 20s")
+ public HudiConfig setColumnStatsWaitTimeout(Duration columnStatusWaitTimeout)
+ {
+ this.columnStatsWaitTimeout = columnStatusWaitTimeout;
+ return this;
+ }
+
+ @NotNull
+ public Duration getColumnStatsWaitTimeout()
+ {
+ return columnStatsWaitTimeout;
+ }
+
+ @Config("hudi.index.record-index.wait-timeout")
+ @ConfigDescription("Maximum timeout to wait for loading record index, e.g. 1000ms, 20s")
+ public HudiConfig setRecordIndexWaitTimeout(Duration recordIndexWaitTimeout)
+ {
+ this.recordIndexWaitTimeout = recordIndexWaitTimeout;
+ return this;
+ }
+
+ @NotNull
+ public Duration getRecordIndexWaitTimeout()
+ {
+ return recordIndexWaitTimeout;
+ }
+
+ @Config("hudi.index.secondary-index.wait-timeout")
+ @ConfigDescription("Maximum timeout to wait for loading secondary index, e.g. 1000ms, 20s")
+ public HudiConfig setSecondaryIndexWaitTimeout(Duration secondaryIndexWaitTimeout)
+ {
+ this.secondaryIndexWaitTimeout = secondaryIndexWaitTimeout;
+ return this;
+ }
+
+ @NotNull
+ public Duration getSecondaryIndexWaitTimeout()
+ {
+ return secondaryIndexWaitTimeout;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnector.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnector.java
new file mode 100644
index 000000000000..072005cd3ffc
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnector.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Injector;
+import io.airlift.bootstrap.LifeCycleManager;
+import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorMetadata;
+import io.trino.plugin.base.session.SessionPropertiesProvider;
+import io.trino.plugin.hive.HiveTransactionHandle;
+import io.trino.spi.classloader.ThreadContextClassLoader;
+import io.trino.spi.connector.Connector;
+import io.trino.spi.connector.ConnectorMetadata;
+import io.trino.spi.connector.ConnectorNodePartitioningProvider;
+import io.trino.spi.connector.ConnectorPageSourceProvider;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorSplitManager;
+import io.trino.spi.connector.ConnectorTransactionHandle;
+import io.trino.spi.connector.SystemTable;
+import io.trino.spi.session.PropertyMetadata;
+import io.trino.spi.transaction.IsolationLevel;
+
+import java.util.List;
+import java.util.Set;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.trino.spi.transaction.IsolationLevel.SERIALIZABLE;
+import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports;
+import static java.util.Objects.requireNonNull;
+
+public class HudiConnector
+ implements Connector
+{
+ private final Injector injector;
+ private final LifeCycleManager lifeCycleManager;
+ private final HudiTransactionManager transactionManager;
+ private final ConnectorSplitManager splitManager;
+ private final ConnectorPageSourceProvider pageSourceProvider;
+ private final ConnectorNodePartitioningProvider nodePartitioningProvider;
+ private final Set<SystemTable> systemTables;
+ private final List<PropertyMetadata<?>> sessionProperties;
+ private final List<PropertyMetadata<?>> tableProperties;
+
+ public HudiConnector(
+ Injector injector,
+ LifeCycleManager lifeCycleManager,
+ HudiTransactionManager transactionManager,
+ ConnectorSplitManager splitManager,
+ ConnectorPageSourceProvider pageSourceProvider,
+ ConnectorNodePartitioningProvider nodePartitioningProvider,
+ Set<SystemTable> systemTables,
+ Set<SessionPropertiesProvider> sessionPropertiesProviders,
+ List<PropertyMetadata<?>> tableProperties)
+ {
+ this.injector = requireNonNull(injector, "injector is null");
+ this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null");
+ this.transactionManager = requireNonNull(transactionManager, "transactionManager is null");
+ this.splitManager = requireNonNull(splitManager, "splitManager is null");
+ this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null");
+ this.nodePartitioningProvider = requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null");
+ this.systemTables = ImmutableSet.copyOf(requireNonNull(systemTables, "systemTables is null"));
+ this.sessionProperties = requireNonNull(sessionPropertiesProviders, "sessionPropertiesProviders is null").stream()
+ .flatMap(sessionPropertiesProvider -> sessionPropertiesProvider.getSessionProperties().stream())
+ .collect(toImmutableList());
+ this.tableProperties = ImmutableList.copyOf(requireNonNull(tableProperties, "tableProperties is null"));
+ }
+
+ @Override
+ public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transactionHandle)
+ {
+ ConnectorMetadata metadata = transactionManager.get(transactionHandle, session.getIdentity());
+ return new ClassLoaderSafeConnectorMetadata(metadata, getClass().getClassLoader());
+ }
+
+ @Override
+ public ConnectorSplitManager getSplitManager()
+ {
+ return splitManager;
+ }
+
+ @Override
+ public ConnectorPageSourceProvider getPageSourceProvider()
+ {
+ return pageSourceProvider;
+ }
+
+ @Override
+ public ConnectorNodePartitioningProvider getNodePartitioningProvider()
+ {
+ return nodePartitioningProvider;
+ }
+
+ @Override
+ public Set<SystemTable> getSystemTables()
+ {
+ return systemTables;
+ }
+
+ @Override
+ public List<PropertyMetadata<?>> getSessionProperties()
+ {
+ return sessionProperties;
+ }
+
+ @Override
+ public List<PropertyMetadata<?>> getTableProperties()
+ {
+ return tableProperties;
+ }
+
+ @Override
+ public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit)
+ {
+ checkConnectorSupports(SERIALIZABLE, isolationLevel);
+ ConnectorTransactionHandle transaction = new HiveTransactionHandle(true);
+ try (ThreadContextClassLoader _ = new ThreadContextClassLoader(getClass().getClassLoader())) {
+ transactionManager.put(transaction);
+ }
+ return transaction;
+ }
+
+ @Override
+ public void commit(ConnectorTransactionHandle transaction)
+ {
+ transactionManager.commit(transaction);
+ }
+
+ @Override
+ public void rollback(ConnectorTransactionHandle transaction)
+ {
+ transactionManager.rollback(transaction);
+ }
+
+ @Override
+ public final void shutdown()
+ {
+ lifeCycleManager.stop();
+ }
+
+ @VisibleForTesting
+ public Injector getInjector()
+ {
+ return injector;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java
new file mode 100644
index 000000000000..f7dd96d76799
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.Module;
+import io.airlift.bootstrap.Bootstrap;
+import io.airlift.bootstrap.LifeCycleManager;
+import io.airlift.json.JsonModule;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.trace.Tracer;
+import io.trino.filesystem.manager.FileSystemModule;
+import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider;
+import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager;
+import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider;
+import io.trino.plugin.base.jmx.MBeanServerModule;
+import io.trino.plugin.base.session.SessionPropertiesProvider;
+import io.trino.plugin.hive.NodeVersion;
+import io.trino.plugin.hive.metastore.HiveMetastoreModule;
+import io.trino.spi.NodeManager;
+import io.trino.spi.catalog.CatalogName;
+import io.trino.spi.classloader.ThreadContextClassLoader;
+import io.trino.spi.connector.Connector;
+import io.trino.spi.connector.ConnectorContext;
+import io.trino.spi.connector.ConnectorFactory;
+import io.trino.spi.connector.ConnectorNodePartitioningProvider;
+import io.trino.spi.connector.ConnectorPageSourceProvider;
+import io.trino.spi.connector.ConnectorSplitManager;
+import io.trino.spi.type.TypeManager;
+import org.weakref.jmx.guice.MBeanModule;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.google.inject.util.Modules.EMPTY_MODULE;
+import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch;
+
+public class HudiConnectorFactory
+ implements ConnectorFactory
+{
+ @Override
+ public String getName()
+ {
+ return "hudi";
+ }
+
+ @Override
+ public Connector create(String catalogName, Map<String, String> config, ConnectorContext context)
+ {
+ checkStrictSpiVersionMatch(context, this);
+ return createConnector(catalogName, config, context, Optional.empty());
+ }
+
+ public static Connector createConnector(
+ String catalogName,
+ Map<String, String> config,
+ ConnectorContext context,
+ Optional<Module> module)
+ {
+ ClassLoader classLoader = HudiConnectorFactory.class.getClassLoader();
+ try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) {
+ Bootstrap app = new Bootstrap(
+ new MBeanModule(),
+ new JsonModule(),
+ new HudiModule(),
+ new HiveMetastoreModule(Optional.empty()),
+ new FileSystemModule(catalogName, context.getNodeManager(), context.getOpenTelemetry(), false),
+ new MBeanServerModule(),
+ module.orElse(EMPTY_MODULE),
+ binder -> {
+ binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry());
+ binder.bind(Tracer.class).toInstance(context.getTracer());
+ binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
+ binder.bind(NodeManager.class).toInstance(context.getNodeManager());
+ binder.bind(TypeManager.class).toInstance(context.getTypeManager());
+ binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName));
+ });
+
+ Injector injector = app
+ .doNotInitializeLogging()
+ .setRequiredConfigurationProperties(config)
+ .initialize();
+
+ LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
+ HudiTransactionManager transactionManager = injector.getInstance(HudiTransactionManager.class);
+ ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
+ ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class);
+ ConnectorNodePartitioningProvider connectorDistributionProvider = injector.getInstance(ConnectorNodePartitioningProvider.class);
+ Set<SessionPropertiesProvider> sessionPropertiesProviders = injector.getInstance(new Key<>() {});
+ HudiTableProperties hudiTableProperties = injector.getInstance(HudiTableProperties.class);
+
+ return new HudiConnector(
+ injector,
+ lifeCycleManager,
+ transactionManager,
+ new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader),
+ new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader),
+ new ClassLoaderSafeNodePartitioningProvider(connectorDistributionProvider, classLoader),
+ ImmutableSet.of(),
+ sessionPropertiesProviders,
+ hudiTableProperties.getTableProperties());
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java
new file mode 100644
index 000000000000..6b34fedc8853
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiErrorCode.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import io.trino.spi.ErrorCode;
+import io.trino.spi.ErrorCodeSupplier;
+import io.trino.spi.ErrorType;
+
+import static io.trino.spi.ErrorType.EXTERNAL;
+
+public enum HudiErrorCode
+ implements ErrorCodeSupplier
+{
+ // HUDI_UNKNOWN_TABLE_TYPE(0, EXTERNAL),
+ HUDI_INVALID_PARTITION_VALUE(1, EXTERNAL),
+ HUDI_BAD_DATA(2, EXTERNAL),
+ // HUDI_MISSING_DATA(3, EXTERNAL) is deprecated
+ HUDI_CANNOT_OPEN_SPLIT(4, EXTERNAL),
+ HUDI_UNSUPPORTED_FILE_FORMAT(5, EXTERNAL),
+ HUDI_CURSOR_ERROR(6, EXTERNAL),
+ HUDI_FILESYSTEM_ERROR(7, EXTERNAL),
+ HUDI_PARTITION_NOT_FOUND(8, EXTERNAL),
+ HUDI_UNSUPPORTED_TABLE_TYPE(9, EXTERNAL),
+ HUDI_NO_VALID_COMMIT(10, EXTERNAL),
+ HUDI_META_CLIENT_ERROR(11, EXTERNAL),
+ /**/;
+
+ private final ErrorCode errorCode;
+
+ HudiErrorCode(int code, ErrorType type)
+ {
+ errorCode = new ErrorCode(code + 0x0507_0000, name(), type);
+ }
+
+ @Override
+ public ErrorCode toErrorCode()
+ {
+ return errorCode;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java
new file mode 100644
index 000000000000..56d585db8772
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiFileStatus.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import io.trino.filesystem.Location;
+
+import static java.util.Objects.requireNonNull;
+
+public record HudiFileStatus(Location location, boolean isDirectory, long length, long modificationTime, long blockSize)
+{
+ public HudiFileStatus
+ {
+ requireNonNull(location, "location is null");
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java
new file mode 100644
index 000000000000..bbe5564b1d15
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadata.java
@@ -0,0 +1,498 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import io.airlift.log.Logger;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystemFactory;
+import io.trino.metastore.Column;
+import io.trino.metastore.HiveMetastore;
+import io.trino.metastore.Table;
+import io.trino.metastore.TableInfo;
+import io.trino.plugin.base.classloader.ClassLoaderSafeSystemTable;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.stats.HudiTableStatistics;
+import io.trino.plugin.hudi.stats.TableStatisticsReader;
+import io.trino.plugin.hudi.util.HudiTableTypeUtils;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ColumnHandle;
+import io.trino.spi.connector.ColumnMetadata;
+import io.trino.spi.connector.ConnectorMetadata;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorTableHandle;
+import io.trino.spi.connector.ConnectorTableMetadata;
+import io.trino.spi.connector.ConnectorTableVersion;
+import io.trino.spi.connector.Constraint;
+import io.trino.spi.connector.ConstraintApplicationResult;
+import io.trino.spi.connector.RelationColumnsMetadata;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.connector.SchemaTablePrefix;
+import io.trino.spi.connector.SystemTable;
+import io.trino.spi.connector.TableNotFoundException;
+import io.trino.spi.predicate.TupleDomain;
+import io.trino.spi.statistics.Estimate;
+import io.trino.spi.statistics.TableStatistics;
+import io.trino.spi.type.TypeManager;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2;
+import org.apache.hudi.common.util.CollectionUtils;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.util.Lazy;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+import java.util.stream.Stream;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static io.trino.metastore.Table.TABLE_COMMENT;
+import static io.trino.plugin.hive.HiveTimestampPrecision.NANOSECONDS;
+import static io.trino.plugin.hive.util.HiveUtil.columnMetadataGetter;
+import static io.trino.plugin.hive.util.HiveUtil.getPartitionKeyColumnHandles;
+import static io.trino.plugin.hive.util.HiveUtil.hiveColumnHandles;
+import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema;
+import static io.trino.plugin.hive.util.HiveUtil.isHudiTable;
+import static io.trino.plugin.hudi.HudiSessionProperties.getColumnsToHide;
+import static io.trino.plugin.hudi.HudiSessionProperties.isHudiMetadataTableEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isQueryPartitionFilterRequired;
+import static io.trino.plugin.hudi.HudiSessionProperties.isTableStatisticsEnabled;
+import static io.trino.plugin.hudi.HudiTableProperties.LOCATION_PROPERTY;
+import static io.trino.plugin.hudi.HudiTableProperties.PARTITIONED_BY_PROPERTY;
+import static io.trino.plugin.hudi.HudiUtil.buildTableMetaClient;
+import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
+import static io.trino.spi.StandardErrorCode.QUERY_REJECTED;
+import static io.trino.spi.StandardErrorCode.UNSUPPORTED_TABLE_TYPE;
+import static io.trino.spi.connector.SchemaTableName.schemaTableName;
+import static java.lang.String.format;
+import static java.util.Collections.singletonList;
+import static java.util.Objects.requireNonNull;
+import static java.util.function.Function.identity;
+import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.CLUSTERING_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.INDEXING_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
+
+public class HudiMetadata
+ implements ConnectorMetadata
+{
+ private static final Logger log = Logger.get(HudiMetadata.class);
+ private static final Map<TableStatisticsCacheKey, HudiTableStatistics> tableStatisticsCache = new ConcurrentHashMap<>();
+ private static final Set<TableStatisticsCacheKey> refreshingKeysInProgress = ConcurrentHashMap.newKeySet();
+ private final HiveMetastore metastore;
+ private final TrinoFileSystemFactory fileSystemFactory;
+ private final TypeManager typeManager;
+ private final ExecutorService tableStatisticsExecutor;
+
+ public HudiMetadata(
+ HiveMetastore metastore,
+ TrinoFileSystemFactory fileSystemFactory,
+ TypeManager typeManager,
+ ExecutorService tableStatisticsExecutor)
+ {
+ this.metastore = requireNonNull(metastore, "metastore is null");
+ this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
+ this.typeManager = requireNonNull(typeManager, "typeManager is null");
+ this.tableStatisticsExecutor = requireNonNull(tableStatisticsExecutor, "tableStatisticsExecutor is null");
+ }
+
+ @Override
+ public List<String> listSchemaNames(ConnectorSession session)
+ {
+ return metastore.getAllDatabases().stream()
+ .filter(schemaName -> !isHiveSystemSchema(schemaName))
+ .collect(toImmutableList());
+ }
+
+ @Override
+ public HudiTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional<ConnectorTableVersion> startVersion, Optional<ConnectorTableVersion> endVersion)
+ {
+ log.info("Creating new HudiTableHandle for %s", tableName);
+ if (startVersion.isPresent() || endVersion.isPresent()) {
+ throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables");
+ }
+
+ if (isHiveSystemSchema(tableName.getSchemaName())) {
+ return null;
+ }
+ Optional<Table> tableOpt = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
+ if (tableOpt.isEmpty()) {
+ return null;
+ }
+
+ Table table = tableOpt.get();
+ if (!isHudiTable(table)) {
+ throw new TrinoException(UNSUPPORTED_TABLE_TYPE, format("Not a Hudi table: %s", tableName));
+ }
+ String basePath = table.getStorage().getLocation();
+ TrinoFileSystem fileSystem = fileSystemFactory.create(session);
+ String inputFormat = table.getStorage().getStorageFormat().getInputFormat();
+ HoodieTableType hoodieTableType = HudiTableTypeUtils.fromInputFormat(inputFormat);
+
+ return new HudiTableHandle(
+ Optional.of(table),
+ Optional.of(Lazy.lazily(() -> buildTableMetaClient(fileSystem, tableName.toString(), basePath))),
+ tableName.getSchemaName(),
+ tableName.getTableName(),
+ table.getStorage().getLocation(),
+ hoodieTableType,
+ getPartitionKeyColumnHandles(table, typeManager),
+ ImmutableSet.of(),
+ TupleDomain.all(),
+ TupleDomain.all());
+ }
+
+ @Override
+ public Optional<SystemTable> getSystemTable(ConnectorSession session, SchemaTableName tableName)
+ {
+ return getRawSystemTable(tableName, session)
+ .map(systemTable -> new ClassLoaderSafeSystemTable(systemTable, getClass().getClassLoader()));
+ }
+
+ private Optional<SystemTable> getRawSystemTable(SchemaTableName tableName, ConnectorSession session)
+ {
+ Optional<HudiTableName> nameOptional = HudiTableName.from(tableName.getTableName());
+ if (nameOptional.isEmpty()) {
+ return Optional.empty();
+ }
+ HudiTableName name = nameOptional.get();
+ if (name.tableType() == TableType.DATA) {
+ return Optional.empty();
+ }
+
+ Optional<Table> tableOptional = metastore.getTable(tableName.getSchemaName(), name.tableName());
+ if (tableOptional.isEmpty()) {
+ return Optional.empty();
+ }
+ if (!isHudiTable(tableOptional.get())) {
+ return Optional.empty();
+ }
+ return switch (name.tableType()) {
+ case DATA -> throw new AssertionError();
+ case TIMELINE -> {
+ SchemaTableName systemTableName = new SchemaTableName(tableName.getSchemaName(), name.tableNameWithType());
+ yield Optional.of(new TimelineTable(fileSystemFactory.create(session), systemTableName, tableOptional.get()));
+ }
+ };
+ }
+
+ @Override
+ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table)
+ {
+ HudiTableHandle hudiTableHandle = (HudiTableHandle) table;
+ return getTableMetadata(hudiTableHandle.getSchemaTableName(), getColumnsToHide(session));
+ }
+
+ @Override
+ public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint)
+ {
+ HudiTableHandle handle = (HudiTableHandle) tableHandle;
+ HudiPredicates predicates = HudiPredicates.from(constraint.getSummary());
+ TupleDomain<HiveColumnHandle> regularColumnPredicates = predicates.getRegularColumnPredicates();
+ TupleDomain<HiveColumnHandle> partitionColumnPredicates = predicates.getPartitionColumnPredicates();
+
+ // TODO Since the constraint#predicate isn't utilized during split generation. So,
+ // Let's not add constraint#predicateColumns to newConstraintColumns.
+ Set<HiveColumnHandle> newConstraintColumns = Stream.concat(
+ Stream.concat(
+ regularColumnPredicates.getDomains().stream()
+ .map(Map::keySet)
+ .flatMap(Collection::stream),
+ partitionColumnPredicates.getDomains().stream()
+ .map(Map::keySet)
+ .flatMap(Collection::stream)),
+ handle.getConstraintColumns().stream())
+ .collect(toImmutableSet());
+
+ HudiTableHandle newHudiTableHandle = handle.applyPredicates(
+ newConstraintColumns,
+ partitionColumnPredicates,
+ regularColumnPredicates);
+
+ if (handle.getPartitionPredicates().equals(newHudiTableHandle.getPartitionPredicates())
+ && handle.getRegularPredicates().equals(newHudiTableHandle.getRegularPredicates())
+ && handle.getConstraintColumns().equals(newHudiTableHandle.getConstraintColumns())) {
+ return Optional.empty();
+ }
+
+ return Optional.of(new ConstraintApplicationResult<>(
+ newHudiTableHandle,
+ newHudiTableHandle.getRegularPredicates().transformKeys(ColumnHandle.class::cast),
+ constraint.getExpression(),
+ false));
+ }
+
+ @Override
+ public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle)
+ {
+ HudiTableHandle hudiTableHandle = (HudiTableHandle) tableHandle;
+ Table table = metastore.getTable(hudiTableHandle.getSchemaName(), hudiTableHandle.getTableName())
+ .orElseThrow(() -> new TableNotFoundException(schemaTableName(hudiTableHandle.getSchemaName(), hudiTableHandle.getTableName())));
+ return hiveColumnHandles(table, typeManager, NANOSECONDS).stream()
+ .collect(toImmutableMap(HiveColumnHandle::getName, identity()));
+ }
+
+ @Override
+ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle)
+ {
+ return ((HiveColumnHandle) columnHandle).getColumnMetadata();
+ }
+
+ @Override
+ public Optional<Object> getInfo(ConnectorTableHandle tableHandle)
+ {
+ HudiTableHandle table = (HudiTableHandle) tableHandle;
+ return Optional.of(new HudiTableInfo(table.getSchemaTableName(), table.getTableType().name(), table.getBasePath()));
+ }
+
+ @Override
+ public List<SchemaTableName> listTables(ConnectorSession session, Optional<String> optionalSchemaName)
+ {
+ ImmutableList.Builder<SchemaTableName> tableNames = ImmutableList.builder();
+ for (String schemaName : listSchemas(session, optionalSchemaName)) {
+ for (TableInfo tableInfo : metastore.getTables(schemaName)) {
+ tableNames.add(tableInfo.tableName());
+ }
+ }
+ return tableNames.build();
+ }
+
+ @Override
+ public Iterator<RelationColumnsMetadata> streamRelationColumns(
+ ConnectorSession session,
+ Optional<String> schemaName,
+ UnaryOperator<Set<SchemaTableName>> relationFilter)
+ {
+ SchemaTablePrefix prefix = schemaName.map(SchemaTablePrefix::new)
+ .orElseGet(SchemaTablePrefix::new);
+ List<SchemaTableName> tables = prefix.getTable()
+ .map(_ -> singletonList(prefix.toSchemaTableName()))
+ .orElseGet(() -> listTables(session, prefix.getSchema()));
+
+ Map<SchemaTableName, RelationColumnsMetadata> relationColumns = tables.stream()
+ .map(table -> getTableColumnMetadata(session, table))
+ .flatMap(Optional::stream)
+ .collect(toImmutableMap(RelationColumnsMetadata::name, Function.identity()));
+ return relationFilter.apply(relationColumns.keySet()).stream()
+ .map(relationColumns::get)
+ .iterator();
+ }
+
+ @Override
+ public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle)
+ {
+ if (!isTableStatisticsEnabled(session) || !isHudiMetadataTableEnabled(session)) {
+ return TableStatistics.empty();
+ }
+
+ List<HiveColumnHandle> columnHandles = getColumnHandles(session, tableHandle)
+ .values().stream()
+ .map(e -> (HiveColumnHandle) e)
+ .filter(e -> !e.isHidden())
+ .toList();
+ return getTableStatisticsFromCache(
+ (HudiTableHandle) tableHandle, columnHandles, tableStatisticsCache, refreshingKeysInProgress, tableStatisticsExecutor);
+ }
+
+ @Override
+ public void validateScan(ConnectorSession session, ConnectorTableHandle handle)
+ {
+ HudiTableHandle hudiTableHandle = (HudiTableHandle) handle;
+ if (isQueryPartitionFilterRequired(session)) {
+ if (!hudiTableHandle.getPartitionColumns().isEmpty()) {
+ Set<String> partitionColumns = hudiTableHandle.getPartitionColumns().stream()
+ .map(HiveColumnHandle::getName)
+ .collect(toImmutableSet());
+ Set<String> constraintColumns = hudiTableHandle.getConstraintColumns().stream()
+ .map(HiveColumnHandle::getBaseColumnName)
+ .collect(toImmutableSet());
+ if (Collections.disjoint(constraintColumns, partitionColumns)) {
+ throw new TrinoException(
+ QUERY_REJECTED,
+ format("Filter required on %s for at least one of the partition columns: %s", hudiTableHandle.getSchemaTableName(), String.join(", ", partitionColumns)));
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean allowSplittingReadIntoMultipleSubQueries(ConnectorSession session, ConnectorTableHandle tableHandle)
+ {
+ // hudi supports only a columnar (parquet) storage format
+ return true;
+ }
+
+ HiveMetastore getMetastore()
+ {
+ return metastore;
+ }
+
+ private Optional<RelationColumnsMetadata> getTableColumnMetadata(ConnectorSession session, SchemaTableName table)
+ {
+ try {
+ List<ColumnMetadata> columns = getTableMetadata(table, getColumnsToHide(session)).getColumns();
+ return Optional.of(RelationColumnsMetadata.forTable(table, columns));
+ }
+ catch (TableNotFoundException _) {
+ return Optional.empty();
+ }
+ }
+
+ private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName, Collection<String> columnsToHide)
+ {
+ Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName())
+ .orElseThrow(() -> new TableNotFoundException(tableName));
+ Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table);
+ List<ColumnMetadata> columns = hiveColumnHandles(table, typeManager, NANOSECONDS).stream()
+ .filter(column -> !columnsToHide.contains(column.getName()))
+ .map(metadataGetter)
+ .collect(toImmutableList());
+
+ ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
+ // Location property
+ String location = table.getStorage().getOptionalLocation().orElse(null);
+ if (!isNullOrEmpty(location)) {
+ properties.put(LOCATION_PROPERTY, location);
+ }
+
+ // Partitioning property
+ List<String> partitionedBy = table.getPartitionColumns().stream()
+ .map(Column::getName)
+ .collect(toImmutableList());
+ if (!partitionedBy.isEmpty()) {
+ properties.put(PARTITIONED_BY_PROPERTY, partitionedBy);
+ }
+
+ Optional<String> comment = Optional.ofNullable(table.getParameters().get(TABLE_COMMENT));
+ return new ConnectorTableMetadata(tableName, columns, properties.buildOrThrow(), comment);
+ }
+
+ private List<String> listSchemas(ConnectorSession session, Optional<String> schemaName)
+ {
+ return schemaName
+ .filter(name -> !isHiveSystemSchema(name))
+ .map(Collections::singletonList)
+ .orElseGet(() -> listSchemaNames(session));
+ }
+
+ private static TableStatistics getTableStatisticsFromCache(
+ HudiTableHandle tableHandle,
+ List<HiveColumnHandle> columnHandles,
+ Map<TableStatisticsCacheKey, HudiTableStatistics> cache,
+ Set<TableStatisticsCacheKey> refreshingKeysInProgress,
+ ExecutorService tableStatisticsExecutor)
+ {
+ TableStatisticsCacheKey key = new TableStatisticsCacheKey(tableHandle.getBasePath());
+ HudiTableStatistics cachedValue = cache.get(key);
+ TableStatistics statisticsToReturn = TableStatistics.empty();
+ if (cachedValue != null) {
+ // Here we avoid checking the latest commit which requires loading the meta client and timeline
+ // which can block query planning. We assume that the cache result might be stale but close
+ // enough for CBO.
+ log.info("Returning cached table statistics for table: %s, latest commit in cache: %s",
+ tableHandle.getSchemaTableName(), cachedValue.latestCommit());
+ statisticsToReturn = cachedValue.tableStatistics();
+ }
+
+ triggerAsyncStatsRefresh(tableHandle, columnHandles, cache, key, refreshingKeysInProgress, tableStatisticsExecutor);
+ return statisticsToReturn;
+ }
+
+ private static void triggerAsyncStatsRefresh(
+ HudiTableHandle tableHandle,
+ List<HiveColumnHandle> columnHandles,
+ Map<TableStatisticsCacheKey, HudiTableStatistics> cache,
+ TableStatisticsCacheKey key,
+ Set<TableStatisticsCacheKey> refreshingKeysInProgress,
+ ExecutorService tableStatisticsExecutor)
+ {
+ if (refreshingKeysInProgress.add(key)) {
+ tableStatisticsExecutor.submit(() -> {
+ HoodieTimer refreshTimer = HoodieTimer.start();
+ try {
+ log.info("Starting async statistics calculation for table: %s", tableHandle.getSchemaTableName());
+ HoodieTableMetaClient metaClient = tableHandle.getMetaClient();
+ Option<HoodieInstant> latestCommitOption = metaClient.getActiveTimeline()
+ .getTimelineOfActions(CollectionUtils.createSet(
+ COMMIT_ACTION, DELTA_COMMIT_ACTION, REPLACE_COMMIT_ACTION, CLUSTERING_ACTION, INDEXING_ACTION))
+ .filterCompletedInstants().lastInstant();
+
+ if (latestCommitOption.isEmpty()) {
+ log.info("Putting table statistics of 0 row in %s ms for empty table: %s",
+ refreshTimer.endTimer(), tableHandle.getSchemaTableName());
+ cache.put(key, new HudiTableStatistics(
+ // A dummy instant that does not match any commit
+ new HoodieInstant(COMPLETED, COMMIT_ACTION, "", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR),
+ TableStatistics.builder().setRowCount(Estimate.of(0)).build()));
+ return;
+ }
+
+ HoodieInstant latestCommit = latestCommitOption.get();
+ HudiTableStatistics oldValue = cache.get(key);
+ if (oldValue != null && latestCommit.equals(oldValue.latestCommit())) {
+ log.info("Table statistics is still valid for table: %s (checked in %s ms)",
+ tableHandle.getSchemaTableName(), refreshTimer.endTimer());
+ return;
+ }
+
+ if (!metaClient.getTableConfig().isMetadataTableAvailable()
+ || !metaClient.getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.COLUMN_STATS)) {
+ log.info("Putting empty table statistics in %s ms as metadata table or "
+ + "column stats is not available for table: %s",
+ refreshTimer.endTimer(), tableHandle.getSchemaTableName());
+ cache.put(key, new HudiTableStatistics(latestCommit, TableStatistics.empty()));
+ return;
+ }
+
+ TableStatistics newStatistics = TableStatisticsReader.create(metaClient)
+ .getTableStatistics(latestCommit, columnHandles);
+ HudiTableStatistics newValue = new HudiTableStatistics(latestCommit, newStatistics);
+ cache.put(key, newValue);
+ log.info("Async table statistics calculation finished in %s ms for table: %s, commit: %s",
+ refreshTimer.endTimer(), tableHandle.getSchemaTableName(), latestCommit);
+ }
+ catch (Throwable e) {
+ log.error(e, "Error calculating table statistics asynchronously for table %s", tableHandle.getSchemaTableName());
+ }
+ finally {
+ refreshingKeysInProgress.remove(key);
+ }
+ });
+ }
+ else {
+ log.debug("Table statistics refresh already in progress for table: %s", tableHandle.getSchemaTableName());
+ }
+ }
+
+ private record TableStatisticsCacheKey(String basePath) {}
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java
new file mode 100644
index 000000000000..ca11b48a61a9
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiMetadataFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.inject.Inject;
+import io.trino.filesystem.TrinoFileSystemFactory;
+import io.trino.metastore.HiveMetastoreFactory;
+import io.trino.metastore.cache.CachingHiveMetastore;
+import io.trino.plugin.hudi.stats.ForHudiTableStatistics;
+import io.trino.spi.security.ConnectorIdentity;
+import io.trino.spi.type.TypeManager;
+
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+
+import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache;
+import static java.util.Objects.requireNonNull;
+
+public class HudiMetadataFactory
+{
+ private final HiveMetastoreFactory metastoreFactory;
+ private final TrinoFileSystemFactory fileSystemFactory;
+ private final TypeManager typeManager;
+ private final long perTransactionMetastoreCacheMaximumSize;
+ private final ExecutorService tableStatisticsExecutor;
+
+ @Inject
+ public HudiMetadataFactory(
+ HiveMetastoreFactory metastoreFactory,
+ TrinoFileSystemFactory fileSystemFactory,
+ TypeManager typeManager,
+ HudiConfig hudiConfig,
+ @ForHudiTableStatistics ExecutorService tableStatisticsExecutor)
+ {
+ this.metastoreFactory = requireNonNull(metastoreFactory, "metastore is null");
+ this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
+ this.typeManager = requireNonNull(typeManager, "typeManager is null");
+ this.perTransactionMetastoreCacheMaximumSize = hudiConfig.getPerTransactionMetastoreCacheMaximumSize();
+ this.tableStatisticsExecutor = requireNonNull(tableStatisticsExecutor, "tableStatisticsExecutor is null");
+ }
+
+ public HudiMetadata create(ConnectorIdentity identity)
+ {
+ CachingHiveMetastore cachingHiveMetastore = createPerTransactionCache(metastoreFactory.createMetastore(Optional.of(identity)), perTransactionMetastoreCacheMaximumSize);
+ return new HudiMetadata(cachingHiveMetastore, fileSystemFactory, typeManager, tableStatisticsExecutor);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java
new file mode 100644
index 000000000000..929cd834c844
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiModule.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.inject.Binder;
+import com.google.inject.Key;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+import io.trino.metastore.HiveMetastore;
+import io.trino.plugin.base.metrics.FileFormatDataSourceStats;
+import io.trino.plugin.base.session.SessionPropertiesProvider;
+import io.trino.plugin.hive.HideDeltaLakeTables;
+import io.trino.plugin.hive.HiveNodePartitioningProvider;
+import io.trino.plugin.hive.HiveTransactionHandle;
+import io.trino.plugin.hive.metastore.thrift.TranslateHiveViews;
+import io.trino.plugin.hive.parquet.ParquetReaderConfig;
+import io.trino.plugin.hive.parquet.ParquetWriterConfig;
+import io.trino.plugin.hudi.stats.ForHudiTableStatistics;
+import io.trino.spi.connector.ConnectorNodePartitioningProvider;
+import io.trino.spi.connector.ConnectorPageSourceProvider;
+import io.trino.spi.connector.ConnectorSplitManager;
+import io.trino.spi.security.ConnectorIdentity;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.BiFunction;
+
+import static com.google.inject.multibindings.Multibinder.newSetBinder;
+import static io.airlift.concurrent.Threads.daemonThreadsNamed;
+import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.trino.plugin.base.ClosingBinder.closingBinder;
+import static java.util.concurrent.Executors.newCachedThreadPool;
+import static java.util.concurrent.Executors.newScheduledThreadPool;
+import static org.weakref.jmx.guice.ExportBinder.newExporter;
+
+public class HudiModule
+ implements Module
+{
+ @Override
+ public void configure(Binder binder)
+ {
+ binder.bind(HudiTransactionManager.class).in(Scopes.SINGLETON);
+
+ configBinder(binder).bindConfig(HudiConfig.class);
+
+ binder.bind(boolean.class).annotatedWith(TranslateHiveViews.class).toInstance(false);
+ binder.bind(boolean.class).annotatedWith(HideDeltaLakeTables.class).toInstance(false);
+
+ newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(HudiSessionProperties.class).in(Scopes.SINGLETON);
+ binder.bind(HudiTableProperties.class).in(Scopes.SINGLETON);
+
+ binder.bind(ConnectorSplitManager.class).to(HudiSplitManager.class).in(Scopes.SINGLETON);
+ binder.bind(ConnectorPageSourceProvider.class).to(HudiPageSourceProvider.class).in(Scopes.SINGLETON);
+ binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class).in(Scopes.SINGLETON);
+
+ configBinder(binder).bindConfig(ParquetReaderConfig.class);
+ configBinder(binder).bindConfig(ParquetWriterConfig.class);
+
+ binder.bind(HudiMetadataFactory.class).in(Scopes.SINGLETON);
+
+ binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(FileFormatDataSourceStats.class).withGeneratedName();
+
+ closingBinder(binder).registerExecutor(Key.get(ExecutorService.class, ForHudiTableStatistics.class));
+ closingBinder(binder).registerExecutor(Key.get(ExecutorService.class, ForHudiSplitManager.class));
+ closingBinder(binder).registerExecutor(Key.get(ScheduledExecutorService.class, ForHudiSplitSource.class));
+ }
+
+ @Provides
+ @Singleton
+ @ForHudiTableStatistics
+ public ExecutorService createTableStatisticsExecutor(HudiConfig hudiConfig)
+ {
+ return newScheduledThreadPool(
+ hudiConfig.getTableStatisticsExecutorParallelism(),
+ daemonThreadsNamed("hudi-table-statistics-executor-%s"));
+ }
+
+ @Provides
+ @Singleton
+ @ForHudiSplitManager
+ public ExecutorService createExecutorService()
+ {
+ return newCachedThreadPool(daemonThreadsNamed("hudi-split-manager-%s"));
+ }
+
+ @Provides
+ @Singleton
+ @ForHudiSplitSource
+ public ScheduledExecutorService createSplitLoaderExecutor(HudiConfig hudiConfig)
+ {
+ return newScheduledThreadPool(
+ hudiConfig.getSplitLoaderParallelism(),
+ daemonThreadsNamed("hudi-split-loader-%s"));
+ }
+
+ @Provides
+ @Singleton
+ public BiFunction<ConnectorIdentity, HiveTransactionHandle, HiveMetastore> createHiveMetastoreGetter(HudiTransactionManager transactionManager)
+ {
+ return (identity, transactionHandle) ->
+ transactionManager.get(transactionHandle, identity).getMetastore();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java
new file mode 100644
index 000000000000..e81887414425
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSource.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.reader.HudiTrinoReaderContext;
+import io.trino.plugin.hudi.util.HudiAvroSerializer;
+import io.trino.plugin.hudi.util.SynthesizedColumnHandler;
+import io.trino.spi.Page;
+import io.trino.spi.PageBuilder;
+import io.trino.spi.connector.ConnectorPageSource;
+import io.trino.spi.metrics.Metrics;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.common.table.read.HoodieFileGroupReader;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.OptionalLong;
+import java.util.concurrent.CompletableFuture;
+
+import static com.google.common.base.Preconditions.checkState;
+
+public class HudiPageSource
+ implements ConnectorPageSource
+{
+ HoodieFileGroupReader<IndexedRecord> fileGroupReader;
+ // TODO: Remove pageSource here, Hudi doesn't use this page source to read
+ ConnectorPageSource pageSource;
+ HudiTrinoReaderContext readerContext;
+ PageBuilder pageBuilder;
+ HudiAvroSerializer avroSerializer;
+ List<HiveColumnHandle> columnHandles;
+
+ public HudiPageSource(
+ ConnectorPageSource pageSource,
+ HoodieFileGroupReader<IndexedRecord> fileGroupReader,
+ HudiTrinoReaderContext readerContext,
+ List<HiveColumnHandle> columnHandles,
+ SynthesizedColumnHandler synthesizedColumnHandler)
+ {
+ this.pageSource = pageSource;
+ this.fileGroupReader = fileGroupReader;
+ this.initFileGroupReader();
+ this.readerContext = readerContext;
+ this.columnHandles = columnHandles;
+ this.pageBuilder = new PageBuilder(columnHandles.stream().map(HiveColumnHandle::getType).toList());
+ this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler);
+ }
+
+ @Override
+ public long getCompletedBytes()
+ {
+ return pageSource.getCompletedBytes();
+ }
+
+ @Override
+ public OptionalLong getCompletedPositions()
+ {
+ return pageSource.getCompletedPositions();
+ }
+
+ @Override
+ public long getReadTimeNanos()
+ {
+ return pageSource.getReadTimeNanos();
+ }
+
+ @Override
+ public boolean isFinished()
+ {
+ try {
+ return !fileGroupReader.hasNext();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Page getNextPage()
+ {
+ checkState(pageBuilder.isEmpty(), "PageBuilder is not empty at the beginning of a new page");
+ try {
+ while (fileGroupReader.hasNext()) {
+ avroSerializer.buildRecordInPage(pageBuilder, fileGroupReader.next());
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ Page newPage = pageBuilder.build();
+ pageBuilder.reset();
+ return newPage;
+ }
+
+ @Override
+ public long getMemoryUsage()
+ {
+ return pageSource.getMemoryUsage();
+ }
+
+ @Override
+ public void close()
+ throws IOException
+ {
+ fileGroupReader.close();
+ pageSource.close();
+ }
+
+ @Override
+ public CompletableFuture<?> isBlocked()
+ {
+ return pageSource.isBlocked();
+ }
+
+ @Override
+ public Metrics getMetrics()
+ {
+ return pageSource.getMetrics();
+ }
+
+ protected void initFileGroupReader()
+ {
+ try {
+ this.fileGroupReader.initRecordIterators();
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Failed to initialize file group reader!", e);
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
new file mode 100644
index 000000000000..500934044b37
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import io.airlift.log.Logger;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystemFactory;
+import io.trino.filesystem.TrinoInputFile;
+import io.trino.memory.context.AggregatedMemoryContext;
+import io.trino.parquet.ParquetCorruptionException;
+import io.trino.parquet.ParquetDataSource;
+import io.trino.parquet.ParquetDataSourceId;
+import io.trino.parquet.ParquetReaderOptions;
+import io.trino.parquet.metadata.FileMetadata;
+import io.trino.parquet.metadata.ParquetMetadata;
+import io.trino.parquet.predicate.TupleDomainParquetPredicate;
+import io.trino.parquet.reader.MetadataReader;
+import io.trino.parquet.reader.ParquetReader;
+import io.trino.parquet.reader.RowGroupInfo;
+import io.trino.plugin.base.metrics.FileFormatDataSourceStats;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.ReaderColumns;
+import io.trino.plugin.hive.parquet.ParquetReaderConfig;
+import io.trino.plugin.hudi.file.HudiBaseFile;
+import io.trino.plugin.hudi.reader.HudiTrinoReaderContext;
+import io.trino.plugin.hudi.storage.HudiTrinoStorage;
+import io.trino.plugin.hudi.storage.TrinoStorageConfiguration;
+import io.trino.plugin.hudi.util.SynthesizedColumnHandler;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ColumnHandle;
+import io.trino.spi.connector.ConnectorPageSource;
+import io.trino.spi.connector.ConnectorPageSourceProvider;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorSplit;
+import io.trino.spi.connector.ConnectorTableHandle;
+import io.trino.spi.connector.ConnectorTransactionHandle;
+import io.trino.spi.connector.DynamicFilter;
+import io.trino.spi.connector.EmptyPageSource;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.TableSchemaResolver;
+import org.apache.hudi.common.table.read.HoodieFileGroupReader;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Type;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.stream.Collectors;
+
+import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext;
+import static io.trino.parquet.ParquetTypeUtils.getColumnIO;
+import static io.trino.parquet.ParquetTypeUtils.getDescriptors;
+import static io.trino.parquet.predicate.PredicateUtils.buildPredicate;
+import static io.trino.parquet.predicate.PredicateUtils.getFilteredRowGroups;
+import static io.trino.plugin.hive.HiveColumnHandle.partitionColumnHandle;
+import static io.trino.plugin.hive.HivePageSourceProvider.projectBaseColumns;
+import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.ParquetReaderProvider;
+import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createDataSource;
+import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.createParquetPageSource;
+import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.getParquetMessageType;
+import static io.trino.plugin.hive.parquet.ParquetPageSourceFactory.getParquetTupleDomain;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_BAD_DATA;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CANNOT_OPEN_SPLIT;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CURSOR_ERROR;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_FILESYSTEM_ERROR;
+import static io.trino.plugin.hudi.HudiSessionProperties.getParquetMaxReadBlockRowCount;
+import static io.trino.plugin.hudi.HudiSessionProperties.getParquetMaxReadBlockSize;
+import static io.trino.plugin.hudi.HudiSessionProperties.getParquetSmallFileThreshold;
+import static io.trino.plugin.hudi.HudiSessionProperties.isParquetIgnoreStatistics;
+import static io.trino.plugin.hudi.HudiSessionProperties.isParquetUseColumnIndex;
+import static io.trino.plugin.hudi.HudiSessionProperties.isParquetVectorizedDecodingEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.shouldUseParquetColumnNames;
+import static io.trino.plugin.hudi.HudiSessionProperties.useParquetBloomFilter;
+import static io.trino.plugin.hudi.HudiUtil.buildTableMetaClient;
+import static io.trino.plugin.hudi.HudiUtil.constructSchema;
+import static io.trino.plugin.hudi.HudiUtil.convertToFileSlice;
+import static io.trino.plugin.hudi.HudiUtil.prependHudiMetaColumns;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toUnmodifiableList;
+
+public class HudiPageSourceProvider
+ implements ConnectorPageSourceProvider
+{
+ private static final Logger log = Logger.get(HudiPageSourceProvider.class);
+ private static final int DOMAIN_COMPACTION_THRESHOLD = 1000;
+
+ private final TrinoFileSystemFactory fileSystemFactory;
+ private final FileFormatDataSourceStats dataSourceStats;
+ private final ParquetReaderOptions options;
+ private final DateTimeZone timeZone = DateTimeZone.forID("UTC");
+
+ @Inject
+ public HudiPageSourceProvider(
+ TrinoFileSystemFactory fileSystemFactory,
+ FileFormatDataSourceStats dataSourceStats,
+ ParquetReaderConfig parquetReaderConfig)
+ {
+ this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
+ this.dataSourceStats = requireNonNull(dataSourceStats, "dataSourceStats is null");
+ this.options = requireNonNull(parquetReaderConfig, "parquetReaderConfig is null").toParquetReaderOptions();
+ }
+
+ @Override
+ public ConnectorPageSource createPageSource(
+ ConnectorTransactionHandle transaction,
+ ConnectorSession session,
+ ConnectorSplit connectorSplit,
+ ConnectorTableHandle connectorTable,
+ List<ColumnHandle> columns,
+ DynamicFilter dynamicFilter)
+ {
+ HudiTableHandle hudiTableHandle = (HudiTableHandle) connectorTable;
+ HudiSplit hudiSplit = (HudiSplit) connectorSplit;
+ Optional<HudiBaseFile> hudiBaseFileOpt = hudiSplit.getBaseFile();
+
+ String dataFilePath = hudiBaseFileOpt.isPresent()
+ ? hudiBaseFileOpt.get().getPath()
+ : hudiSplit.getLogFiles().getFirst().getPath();
+ // Filter out metadata table splits
+ // TODO: Move this check into a higher calling stack, such that the split is not created at all
+ if (dataFilePath.contains(new StoragePath(
+ ((HudiTableHandle) connectorTable).getBasePath()).toUri().getPath() + "/.hoodie/metadata")) {
+ return new EmptyPageSource();
+ }
+
+ // Handle MERGE_ON_READ tables to be read in read_optimized mode
+ // IMPORTANT: These tables will have a COPY_ON_WRITE table, see: `HudiTableTypeUtils#fromInputFormat`
+ // TODO: Move this check into a higher calling stack, such that the split is not created at all
+ if (hudiTableHandle.getTableType().equals(HoodieTableType.COPY_ON_WRITE) && !hudiSplit.getLogFiles().isEmpty()) {
+ if (hudiBaseFileOpt.isEmpty()) {
+ // Handle hasLogFiles=true, hasBaseFile = false
+ // Ignoring log files without base files, no data required to be read
+ return new EmptyPageSource();
+ }
+ }
+
+ long start = 0;
+ long length = 10;
+ if (hudiBaseFileOpt.isPresent()) {
+ start = hudiBaseFileOpt.get().getStart();
+ length = hudiBaseFileOpt.get().getLength();
+ }
+
+ // Enable predicate pushdown for splits containing only base files
+ boolean isBaseFileOnly = hudiSplit.getLogFiles().isEmpty();
+ // Convert columns to HiveColumnHandles
+ List<HiveColumnHandle> hiveColumnHandles = getHiveColumns(columns, isBaseFileOnly);
+
+ // Get non-synthesized columns (columns that are available in data file)
+ List<HiveColumnHandle> dataColumnHandles = hiveColumnHandles.stream()
+ .filter(columnHandle -> !columnHandle.isPartitionKey() && !columnHandle.isHidden())
+ .collect(Collectors.toList());
+ // The `columns` list could be empty when count(*) is issued,
+ // prepending hoodie meta columns for Hudi split with log files
+ // to allow a non-empty dataPageSource to be returned
+ List<HiveColumnHandle> hudiMetaAndDataColumnHandles = prependHudiMetaColumns(dataColumnHandles);
+
+ TrinoFileSystem fileSystem = fileSystemFactory.create(session);
+ ConnectorPageSource dataPageSource = createPageSource(
+ session,
+ isBaseFileOnly ? dataColumnHandles : hudiMetaAndDataColumnHandles,
+ hudiSplit,
+ fileSystem.newInputFile(Location.of(hudiBaseFileOpt.get().getPath()), hudiBaseFileOpt.get().getFileSize()),
+ dataSourceStats,
+ options
+ .withIgnoreStatistics(isParquetIgnoreStatistics(session))
+ .withMaxReadBlockSize(getParquetMaxReadBlockSize(session))
+ .withMaxReadBlockRowCount(getParquetMaxReadBlockRowCount(session))
+ .withSmallFileThreshold(getParquetSmallFileThreshold(session))
+ .withUseColumnIndex(isParquetUseColumnIndex(session))
+ .withBloomFilter(useParquetBloomFilter(session))
+ .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)),
+ timeZone, dynamicFilter, isBaseFileOnly);
+
+ SynthesizedColumnHandler synthesizedColumnHandler = SynthesizedColumnHandler.create(hudiSplit);
+
+ // Avoid avro serialization if split/filegroup only contains base files
+ if (isBaseFileOnly) {
+ ValidationUtils.checkArgument(!hiveColumnHandles.isEmpty(),
+ "Column handles should always be present for providing Hudi data page source on a base file");
+ return new HudiBaseFileOnlyPageSource(
+ dataPageSource,
+ hiveColumnHandles,
+ dataColumnHandles,
+ synthesizedColumnHandler);
+ }
+
+ // TODO: Move this into HudiTableHandle
+ HoodieTableMetaClient metaClient = buildTableMetaClient(
+ fileSystemFactory.create(session), hudiTableHandle.getSchemaTableName().toString(), hudiTableHandle.getBasePath());
+ String latestCommitTime = metaClient.getCommitsTimeline().lastInstant().get().requestedTime();
+ Schema dataSchema;
+ try {
+ dataSchema = new TableSchemaResolver(metaClient).getTableAvroSchema(latestCommitTime);
+ }
+ catch (Throwable e) {
+ // Unable to find table schema
+ throw new TrinoException(HUDI_FILESYSTEM_ERROR, e);
+ }
+
+ HudiTrinoReaderContext readerContext = new HudiTrinoReaderContext(
+ dataPageSource,
+ dataColumnHandles,
+ hudiMetaAndDataColumnHandles,
+ synthesizedColumnHandler);
+
+ // Construct an Avro schema for log file reader
+ Schema requestedSchema = constructSchema(dataSchema, hudiMetaAndDataColumnHandles.stream().map(HiveColumnHandle::getName).toList());
+ HoodieFileGroupReader<IndexedRecord> fileGroupReader =
+ new HoodieFileGroupReader<>(
+ readerContext,
+ new HudiTrinoStorage(fileSystemFactory.create(session), new TrinoStorageConfiguration()),
+ hudiTableHandle.getBasePath(),
+ latestCommitTime,
+ convertToFileSlice(hudiSplit, hudiTableHandle.getBasePath()),
+ dataSchema,
+ requestedSchema,
+ Option.empty(),
+ metaClient,
+ metaClient.getTableConfig().getProps(),
+ start,
+ length,
+ false);
+
+ return new HudiPageSource(
+ dataPageSource,
+ fileGroupReader,
+ readerContext,
+ hiveColumnHandles,
+ synthesizedColumnHandler);
+ }
+
+ static ConnectorPageSource createPageSource(
+ ConnectorSession session,
+ List<HiveColumnHandle> columns,
+ HudiSplit hudiSplit,
+ TrinoInputFile inputFile,
+ FileFormatDataSourceStats dataSourceStats,
+ ParquetReaderOptions options,
+ DateTimeZone timeZone,
+ DynamicFilter dynamicFilter,
+ boolean enablePredicatePushDown)
+ {
+ ParquetDataSource dataSource = null;
+ boolean useColumnNames = shouldUseParquetColumnNames(session);
+ HudiBaseFile baseFile = hudiSplit.getBaseFile().get();
+ String path = baseFile.getPath();
+ long start = baseFile.getStart();
+ long length = baseFile.getLength();
+ try {
+ AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext();
+ dataSource = createDataSource(inputFile, OptionalLong.of(baseFile.getFileSize()), options, memoryContext, dataSourceStats);
+ ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty());
+ FileMetadata fileMetaData = parquetMetadata.getFileMetaData();
+ MessageType fileSchema = fileMetaData.getSchema();
+
+ // When not using columnNames, physical indexes are used and there could be cases when the physical index in HiveColumnHandle is different from the fileSchema of the
+ // parquet files. This could happen when schema evolution happened. In such a case, we will need to remap the column indices in the HiveColumnHandles.
+ if (!useColumnNames) {
+ // HiveColumnHandle names are in lower case, case-insensitive
+ columns = remapColumnIndicesToPhysical(fileSchema, columns, false);
+ }
+
+ Optional<MessageType> message = getParquetMessageType(columns, useColumnNames, fileSchema);
+
+ MessageType requestedSchema = message.orElse(new MessageType(fileSchema.getName(), ImmutableList.of()));
+ MessageColumnIO messageColumn = getColumnIO(fileSchema, requestedSchema);
+
+ Map<List<String>, ColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
+
+ TupleDomain<ColumnDescriptor> parquetTupleDomain = options.isIgnoreStatistics() || !enablePredicatePushDown
+ ? TupleDomain.all()
+ : getParquetTupleDomain(descriptorsByPath, getCombinedPredicate(hudiSplit, dynamicFilter), fileSchema, useColumnNames);
+
+ TupleDomainParquetPredicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath, timeZone);
+
+ List<RowGroupInfo> rowGroups = getFilteredRowGroups(
+ start,
+ length,
+ dataSource,
+ parquetMetadata,
+ ImmutableList.of(parquetTupleDomain),
+ ImmutableList.of(parquetPredicate),
+ descriptorsByPath,
+ timeZone,
+ DOMAIN_COMPACTION_THRESHOLD,
+ options);
+
+ Optional<ReaderColumns> readerProjections = projectBaseColumns(columns);
+ List<HiveColumnHandle> baseColumns = readerProjections.map(projection ->
+ projection.get().stream()
+ .map(HiveColumnHandle.class::cast)
+ .collect(toUnmodifiableList()))
+ .orElse(columns);
+ ParquetDataSourceId dataSourceId = dataSource.getId();
+ ParquetDataSource finalDataSource = dataSource;
+ ParquetReaderProvider parquetReaderProvider = fields -> new ParquetReader(
+ Optional.ofNullable(fileMetaData.getCreatedBy()),
+ fields,
+ rowGroups,
+ finalDataSource,
+ timeZone,
+ memoryContext,
+ options,
+ exception -> handleException(dataSourceId, exception),
+ Optional.of(parquetPredicate),
+ Optional.empty());
+ return createParquetPageSource(baseColumns, fileSchema, messageColumn, useColumnNames, parquetReaderProvider);
+ }
+ catch (IOException | RuntimeException e) {
+ try {
+ if (dataSource != null) {
+ dataSource.close();
+ }
+ }
+ catch (IOException _) {
+ }
+ if (e instanceof TrinoException) {
+ throw (TrinoException) e;
+ }
+ if (e instanceof ParquetCorruptionException) {
+ throw new TrinoException(HUDI_BAD_DATA, e);
+ }
+ String message = "Error opening Hudi split %s (offset=%s, length=%s): %s".formatted(path, start, length, e.getMessage());
+ throw new TrinoException(HUDI_CANNOT_OPEN_SPLIT, message, e);
+ }
+ }
+
+ private static TrinoException handleException(ParquetDataSourceId dataSourceId, Exception exception)
+ {
+ if (exception instanceof TrinoException) {
+ return (TrinoException) exception;
+ }
+ if (exception instanceof ParquetCorruptionException) {
+ return new TrinoException(HUDI_BAD_DATA, exception);
+ }
+ return new TrinoException(HUDI_CURSOR_ERROR, format("Failed to read Parquet file: %s", dataSourceId), exception);
+ }
+
+ /**
+ * Creates a new list of ColumnHandles where the index associated with each handle corresponds to its physical position within the provided fileSchema (MessageType).
+ * This is necessary when a downstream component relies on the handle's index for physical data access, and the logical schema order (potentially reflected in the
+ * original handles) differs from the physical file layout.
+ *
+ * @param fileSchema The MessageType representing the physical schema of the Parquet file.
+ * @param requestedColumns The original list of Trino ColumnHandles as received from the engine.
+ * @param caseSensitive Whether the lookup between Trino column names (from handles) and Parquet field names (from fileSchema) should be case-sensitive.
+ * @return A new list of HiveColumnHandle, preserving the original order, but with each handle containing the correct physical index relative to fileSchema.
+ */
+ @VisibleForTesting
+ public static List<HiveColumnHandle> remapColumnIndicesToPhysical(
+ MessageType fileSchema,
+ List<HiveColumnHandle> requestedColumns,
+ boolean caseSensitive)
+ {
+ // Create a map from column name to its physical index in the fileSchema.
+ Map<String, Integer> physicalIndexMap = new HashMap<>();
+ List<Type> fileFields = fileSchema.getFields();
+ for (int i = 0; i < fileFields.size(); i++) {
+ Type field = fileFields.get(i);
+ String fieldName = field.getName();
+ String mapKey = caseSensitive ? fieldName : fieldName.toLowerCase(Locale.getDefault());
+ physicalIndexMap.put(mapKey, i);
+ }
+
+ // Iterate through the columns requested by Trino IN ORDER.
+ List<HiveColumnHandle> remappedHandles = new ArrayList<>(requestedColumns.size());
+ for (HiveColumnHandle originalHandle : requestedColumns) {
+ String requestedName = originalHandle.getBaseColumnName();
+
+ // Determine the key to use for looking up the physical index
+ String lookupKey = caseSensitive ? requestedName : requestedName.toLowerCase(Locale.getDefault());
+
+ // Find the physical index from the file schema map constructed from fielSchema
+ Integer physicalIndex = physicalIndexMap.get(lookupKey);
+
+ HiveColumnHandle remappedHandle = new HiveColumnHandle(
+ requestedName,
+ physicalIndex,
+ originalHandle.getBaseHiveType(),
+ originalHandle.getType(),
+ originalHandle.getHiveColumnProjectionInfo(),
+ originalHandle.getColumnType(),
+ originalHandle.getComment());
+ remappedHandles.add(remappedHandle);
+ }
+
+ return remappedHandles;
+ }
+
+ private static TupleDomain<HiveColumnHandle> getCombinedPredicate(HudiSplit hudiSplit, DynamicFilter dynamicFilter)
+ {
+ // Combine static and dynamic predicates
+ TupleDomain<HiveColumnHandle> staticPredicate = hudiSplit.getPredicate();
+ TupleDomain<HiveColumnHandle> dynamicPredicate = dynamicFilter.getCurrentPredicate()
+ .transformKeys(HiveColumnHandle.class::cast);
+ TupleDomain<HiveColumnHandle> combinedPredicate = staticPredicate.intersect(dynamicPredicate);
+
+ if (!combinedPredicate.isAll()) {
+ log.debug("Combined predicate for Parquet read (Split: %s): %s", hudiSplit, combinedPredicate);
+ }
+ return combinedPredicate;
+ }
+
+ private static List<HiveColumnHandle> getHiveColumns(List<ColumnHandle> columns,
+ boolean isBaseFileOnly)
+ {
+ if (!isBaseFileOnly || !columns.isEmpty()) {
+ return columns.stream()
+ .map(HiveColumnHandle.class::cast)
+ .toList();
+ }
+
+ // The `columns` list containing the requested columns to read could be empty
+ // when count(*) is in the statement; to make sure the page source works properly,
+ // the synthesized partition column is added in this case.
+ return Collections.singletonList(partitionColumnHandle());
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPlugin.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPlugin.java
new file mode 100644
index 000000000000..1744105faddb
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPlugin.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import io.trino.spi.Plugin;
+import io.trino.spi.connector.ConnectorFactory;
+
+public class HudiPlugin
+ implements Plugin
+{
+ @Override
+ public Iterable<ConnectorFactory> getConnectorFactories()
+ {
+ return ImmutableList.of(new HudiConnectorFactory());
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPredicates.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPredicates.java
new file mode 100644
index 000000000000..16859e8e6dd3
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiPredicates.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.spi.connector.ColumnHandle;
+import io.trino.spi.predicate.Domain;
+import io.trino.spi.predicate.TupleDomain;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class HudiPredicates
+{
+ private final TupleDomain<HiveColumnHandle> partitionColumnPredicates;
+ private final TupleDomain<HiveColumnHandle> regularColumnPredicates;
+
+ public static HudiPredicates from(TupleDomain<ColumnHandle> predicate)
+ {
+ Map<HiveColumnHandle, Domain> partitionColumnPredicates = new HashMap<>();
+ Map<HiveColumnHandle, Domain> regularColumnPredicates = new HashMap<>();
+
+ Optional<Map<ColumnHandle, Domain>> domains = predicate.getDomains();
+ domains.ifPresent(columnHandleDomainMap -> columnHandleDomainMap.forEach((key, value) -> {
+ HiveColumnHandle columnHandle = (HiveColumnHandle) key;
+ if (columnHandle.isPartitionKey()) {
+ partitionColumnPredicates.put(columnHandle, value);
+ }
+ else {
+ regularColumnPredicates.put(columnHandle, value);
+ }
+ }));
+
+ return new HudiPredicates(
+ TupleDomain.withColumnDomains(partitionColumnPredicates),
+ TupleDomain.withColumnDomains(regularColumnPredicates));
+ }
+
+ private HudiPredicates(
+ TupleDomain<HiveColumnHandle> partitionColumnPredicates,
+ TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ this.partitionColumnPredicates = partitionColumnPredicates;
+ this.regularColumnPredicates = regularColumnPredicates;
+ }
+
+ public TupleDomain<HiveColumnHandle> getPartitionColumnPredicates()
+ {
+ return partitionColumnPredicates;
+ }
+
+ public TupleDomain<HiveColumnHandle> getRegularColumnPredicates()
+ {
+ return regularColumnPredicates;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
new file mode 100644
index 000000000000..c1414f13ee89
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSessionProperties.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import io.airlift.units.DataSize;
+import io.airlift.units.Duration;
+import io.trino.plugin.base.session.SessionPropertiesProvider;
+import io.trino.plugin.hive.parquet.ParquetReaderConfig;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.session.PropertyMetadata;
+import io.trino.spi.type.ArrayType;
+
+import java.util.Collection;
+import java.util.List;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.trino.plugin.base.session.PropertyMetadataUtil.dataSizeProperty;
+import static io.trino.plugin.base.session.PropertyMetadataUtil.durationProperty;
+import static io.trino.plugin.base.session.PropertyMetadataUtil.validateMaxDataSize;
+import static io.trino.plugin.hive.parquet.ParquetReaderConfig.PARQUET_READER_MAX_SMALL_FILE_THRESHOLD;
+import static io.trino.spi.StandardErrorCode.INVALID_SESSION_PROPERTY;
+import static io.trino.spi.session.PropertyMetadata.booleanProperty;
+import static io.trino.spi.session.PropertyMetadata.doubleProperty;
+import static io.trino.spi.session.PropertyMetadata.integerProperty;
+import static io.trino.spi.type.VarcharType.VARCHAR;
+import static java.lang.String.format;
+import static java.util.Locale.ENGLISH;
+
+public class HudiSessionProperties
+ implements SessionPropertiesProvider
+{
+ private static final String COLUMNS_TO_HIDE = "columns_to_hide";
+ static final String TABLE_STATISTICS_ENABLED = "table_statistics_enabled";
+ static final String METADATA_TABLE_ENABLED = "metadata_enabled";
+ private static final String USE_PARQUET_COLUMN_NAMES = "use_parquet_column_names";
+ private static final String PARQUET_IGNORE_STATISTICS = "parquet_ignore_statistics";
+ private static final String PARQUET_USE_COLUMN_INDEX = "parquet_use_column_index";
+ private static final String PARQUET_USE_BLOOM_FILTER = "parquet_use_bloom_filter";
+ private static final String PARQUET_MAX_READ_BLOCK_SIZE = "parquet_max_read_block_size";
+ private static final String PARQUET_MAX_READ_BLOCK_ROW_COUNT = "parquet_max_read_block_row_count";
+ private static final String PARQUET_SMALL_FILE_THRESHOLD = "parquet_small_file_threshold";
+ private static final String PARQUET_VECTORIZED_DECODING_ENABLED = "parquet_vectorized_decoding_enabled";
+ private static final String SIZE_BASED_SPLIT_WEIGHTS_ENABLED = "size_based_split_weights_enabled";
+ private static final String STANDARD_SPLIT_WEIGHT_SIZE = "standard_split_weight_size";
+ private static final String MINIMUM_ASSIGNED_SPLIT_WEIGHT = "minimum_assigned_split_weight";
+ private static final String TARGET_SPLIT_SIZE = "target_split_size";
+ private static final String MAX_SPLITS_PER_SECOND = "max_splits_per_second";
+ private static final String MAX_OUTSTANDING_SPLITS = "max_outstanding_splits";
+ private static final String SPLIT_GENERATOR_PARALLELISM = "split_generator_parallelism";
+ static final String QUERY_PARTITION_FILTER_REQUIRED = "query_partition_filter_required";
+ private static final String IGNORE_ABSENT_PARTITIONS = "ignore_absent_partitions";
+ static final String DYNAMIC_FILTERING_WAIT_TIMEOUT = "dynamic_filtering_wait_timeout";
+
+ // Internal configuration for debugging and testing
+ static final String RECORD_LEVEL_INDEX_ENABLED = "record_level_index_enabled";
+ static final String SECONDARY_INDEX_ENABLED = "secondary_index_enabled";
+ static final String COLUMN_STATS_INDEX_ENABLED = "column_stats_index_enabled";
+ static final String PARTITION_STATS_INDEX_ENABLED = "partition_stats_index_enabled";
+ static final String COLUMN_STATS_WAIT_TIMEOUT = "column_stats_wait_timeout";
+ static final String RECORD_INDEX_WAIT_TIMEOUT = "record_index_wait_timeout";
+ static final String SECONDARY_INDEX_WAIT_TIMEOUT = "secondary_index_wait_timeout";
+
+ private final List<PropertyMetadata<?>> sessionProperties;
+
+ @Inject
+ public HudiSessionProperties(HudiConfig hudiConfig, ParquetReaderConfig parquetReaderConfig)
+ {
+ sessionProperties = ImmutableList.of(
+ new PropertyMetadata<>(
+ COLUMNS_TO_HIDE,
+ "List of column names that will be hidden",
+ new ArrayType(VARCHAR),
+ List.class,
+ hudiConfig.getColumnsToHide(),
+ false,
+ value -> ((Collection<?>) value).stream()
+ .map(name -> ((String) name).toLowerCase(ENGLISH))
+ .collect(toImmutableList()),
+ value -> value),
+ booleanProperty(
+ TABLE_STATISTICS_ENABLED,
+ "Expose table statistics",
+ hudiConfig.isTableStatisticsEnabled(),
+ false),
+ booleanProperty(
+ METADATA_TABLE_ENABLED,
+ "For Hudi tables prefer to fetch the list of files from its metadata table",
+ hudiConfig.isMetadataEnabled(),
+ false),
+ booleanProperty(
+ USE_PARQUET_COLUMN_NAMES,
+ "Access parquet columns using names from the file. If disabled, then columns are accessed using index.",
+ hudiConfig.getUseParquetColumnNames(),
+ false),
+ booleanProperty(
+ PARQUET_IGNORE_STATISTICS,
+ "Ignore statistics from Parquet to allow querying files with corrupted or incorrect statistics",
+ parquetReaderConfig.isIgnoreStatistics(),
+ false),
+ booleanProperty(
+ PARQUET_USE_COLUMN_INDEX,
+ "Use Parquet column index",
+ hudiConfig.isUseParquetColumnIndex(),
+ false),
+ booleanProperty(
+ PARQUET_USE_BLOOM_FILTER,
+ "Use Parquet Bloom filters",
+ parquetReaderConfig.isUseBloomFilter(),
+ false),
+ dataSizeProperty(
+ PARQUET_MAX_READ_BLOCK_SIZE,
+ "Parquet: Maximum size of a block to read",
+ parquetReaderConfig.getMaxReadBlockSize(),
+ false),
+ integerProperty(
+ PARQUET_MAX_READ_BLOCK_ROW_COUNT,
+ "Parquet: Maximum number of rows read in a batch",
+ parquetReaderConfig.getMaxReadBlockRowCount(),
+ value -> {
+ if (value < 128 || value > 65536) {
+ throw new TrinoException(
+ INVALID_SESSION_PROPERTY,
+ format("%s must be between 128 and 65536: %s", PARQUET_MAX_READ_BLOCK_ROW_COUNT, value));
+ }
+ },
+ false),
+ dataSizeProperty(
+ PARQUET_SMALL_FILE_THRESHOLD,
+ "Parquet: Size below which a parquet file will be read entirely",
+ parquetReaderConfig.getSmallFileThreshold(),
+ value -> validateMaxDataSize(PARQUET_SMALL_FILE_THRESHOLD, value, DataSize.valueOf(PARQUET_READER_MAX_SMALL_FILE_THRESHOLD)),
+ false),
+ booleanProperty(
+ PARQUET_VECTORIZED_DECODING_ENABLED,
+ "Enable using Java Vector API for faster decoding of parquet files",
+ parquetReaderConfig.isVectorizedDecodingEnabled(),
+ false),
+ booleanProperty(
+ SIZE_BASED_SPLIT_WEIGHTS_ENABLED,
+ format("If enabled, size-based splitting ensures that each batch of splits has enough data to process as defined by %s", STANDARD_SPLIT_WEIGHT_SIZE),
+ hudiConfig.isSizeBasedSplitWeightsEnabled(),
+ false),
+ dataSizeProperty(
+ STANDARD_SPLIT_WEIGHT_SIZE,
+ "The split size corresponding to the standard weight (1.0) when size-based split weights are enabled",
+ hudiConfig.getStandardSplitWeightSize(),
+ false),
+ doubleProperty(
+ MINIMUM_ASSIGNED_SPLIT_WEIGHT,
+ "Minimum assigned split weight when size-based split weights are enabled",
+ hudiConfig.getMinimumAssignedSplitWeight(),
+ value -> {
+ if (!Double.isFinite(value) || value <= 0 || value > 1) {
+ throw new TrinoException(INVALID_SESSION_PROPERTY, format("%s must be > 0 and <= 1.0: %s", MINIMUM_ASSIGNED_SPLIT_WEIGHT, value));
+ }
+ },
+ false),
+ dataSizeProperty(
+ TARGET_SPLIT_SIZE,
+ "The target split size",
+ hudiConfig.getTargetSplitSize(),
+ false),
+ integerProperty(
+ MAX_SPLITS_PER_SECOND,
+ "Rate at which splits are enqueued for processing. The queue will throttle if this rate limit is breached.",
+ hudiConfig.getMaxSplitsPerSecond(),
+ false),
+ integerProperty(
+ MAX_OUTSTANDING_SPLITS,
+ "Maximum outstanding splits in a batch enqueued for processing",
+ hudiConfig.getMaxOutstandingSplits(),
+ false),
+ integerProperty(
+ SPLIT_GENERATOR_PARALLELISM,
+ "Number of threads to generate splits from partitions",
+ hudiConfig.getSplitGeneratorParallelism(),
+ false),
+ booleanProperty(
+ QUERY_PARTITION_FILTER_REQUIRED,
+ "Require a filter on at least one partition column",
+ hudiConfig.isQueryPartitionFilterRequired(),
+ false),
+ booleanProperty(
+ IGNORE_ABSENT_PARTITIONS,
+ "Ignore absent partitions",
+ hudiConfig.isIgnoreAbsentPartitions(),
+ false),
+ booleanProperty(
+ RECORD_LEVEL_INDEX_ENABLED,
+ "Enable record level index for file skipping",
+ hudiConfig.isRecordLevelIndexEnabled(),
+ true),
+ booleanProperty(
+ SECONDARY_INDEX_ENABLED,
+ "Enable secondary index for file skipping",
+ hudiConfig.isSecondaryIndexEnabled(),
+ true),
+ booleanProperty(
+ COLUMN_STATS_INDEX_ENABLED,
+ "Enable column stats index for file skipping",
+ hudiConfig.isColumnStatsIndexEnabled(),
+ true),
+ booleanProperty(
+ PARTITION_STATS_INDEX_ENABLED,
+ "Enable partition stats index for file skipping",
+ hudiConfig.isPartitionStatsIndexEnabled(),
+ true),
+ durationProperty(
+ COLUMN_STATS_WAIT_TIMEOUT,
+ "Maximum timeout to wait for loading column stats",
+ hudiConfig.getColumnStatsWaitTimeout(),
+ false),
+ durationProperty(
+ RECORD_INDEX_WAIT_TIMEOUT,
+ "Maximum timeout to wait for loading record index",
+ hudiConfig.getRecordIndexWaitTimeout(),
+ false),
+ durationProperty(
+ SECONDARY_INDEX_WAIT_TIMEOUT,
+ "Maximum timeout to wait for loading secondary index",
+ hudiConfig.getSecondaryIndexWaitTimeout(),
+ false),
+ durationProperty(
+ DYNAMIC_FILTERING_WAIT_TIMEOUT,
+ "Duration to wait for completion of dynamic filters during split generation",
+ hudiConfig.getDynamicFilteringWaitTimeout(),
+ false));
+ }
+
+ @Override
+ public List<PropertyMetadata<?>> getSessionProperties()
+ {
+ return sessionProperties;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static List<String> getColumnsToHide(ConnectorSession session)
+ {
+ return (List<String>) session.getProperty(COLUMNS_TO_HIDE, List.class);
+ }
+
+ public static boolean isTableStatisticsEnabled(ConnectorSession session)
+ {
+ return session.getProperty(TABLE_STATISTICS_ENABLED, Boolean.class);
+ }
+
+ public static boolean isHudiMetadataTableEnabled(ConnectorSession session)
+ {
+ return session.getProperty(METADATA_TABLE_ENABLED, Boolean.class);
+ }
+
+ public static boolean shouldUseParquetColumnNames(ConnectorSession session)
+ {
+ return session.getProperty(USE_PARQUET_COLUMN_NAMES, Boolean.class);
+ }
+
+ public static boolean isParquetIgnoreStatistics(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_IGNORE_STATISTICS, Boolean.class);
+ }
+
+ public static boolean isParquetUseColumnIndex(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_USE_COLUMN_INDEX, Boolean.class);
+ }
+
+ public static boolean useParquetBloomFilter(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_USE_BLOOM_FILTER, Boolean.class);
+ }
+
+ public static DataSize getParquetMaxReadBlockSize(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_MAX_READ_BLOCK_SIZE, DataSize.class);
+ }
+
+ public static int getParquetMaxReadBlockRowCount(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_MAX_READ_BLOCK_ROW_COUNT, Integer.class);
+ }
+
+ public static DataSize getParquetSmallFileThreshold(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_SMALL_FILE_THRESHOLD, DataSize.class);
+ }
+
+ public static boolean isParquetVectorizedDecodingEnabled(ConnectorSession session)
+ {
+ return session.getProperty(PARQUET_VECTORIZED_DECODING_ENABLED, Boolean.class);
+ }
+
+ public static boolean isSizeBasedSplitWeightsEnabled(ConnectorSession session)
+ {
+ return session.getProperty(SIZE_BASED_SPLIT_WEIGHTS_ENABLED, Boolean.class);
+ }
+
+ public static DataSize getStandardSplitWeightSize(ConnectorSession session)
+ {
+ return session.getProperty(STANDARD_SPLIT_WEIGHT_SIZE, DataSize.class);
+ }
+
+ public static double getMinimumAssignedSplitWeight(ConnectorSession session)
+ {
+ return session.getProperty(MINIMUM_ASSIGNED_SPLIT_WEIGHT, Double.class);
+ }
+
+ public static DataSize getTargetSplitSize(ConnectorSession session)
+ {
+ return session.getProperty(TARGET_SPLIT_SIZE, DataSize.class);
+ }
+
+ public static int getMaxSplitsPerSecond(ConnectorSession session)
+ {
+ return session.getProperty(MAX_SPLITS_PER_SECOND, Integer.class);
+ }
+
+ public static int getMaxOutstandingSplits(ConnectorSession session)
+ {
+ return session.getProperty(MAX_OUTSTANDING_SPLITS, Integer.class);
+ }
+
+ public static int getSplitGeneratorParallelism(ConnectorSession session)
+ {
+ return session.getProperty(SPLIT_GENERATOR_PARALLELISM, Integer.class);
+ }
+
+ public static boolean isQueryPartitionFilterRequired(ConnectorSession session)
+ {
+ return session.getProperty(QUERY_PARTITION_FILTER_REQUIRED, Boolean.class);
+ }
+
+ public static boolean isIgnoreAbsentPartitions(ConnectorSession session)
+ {
+ return session.getProperty(IGNORE_ABSENT_PARTITIONS, Boolean.class);
+ }
+
+ public static boolean isRecordLevelIndexEnabled(ConnectorSession session)
+ {
+ return session.getProperty(RECORD_LEVEL_INDEX_ENABLED, Boolean.class);
+ }
+
+ public static boolean isSecondaryIndexEnabled(ConnectorSession session)
+ {
+ return session.getProperty(SECONDARY_INDEX_ENABLED, Boolean.class);
+ }
+
+ public static boolean isColumnStatsIndexEnabled(ConnectorSession session)
+ {
+ return session.getProperty(COLUMN_STATS_INDEX_ENABLED, Boolean.class);
+ }
+
+ public static boolean isPartitionStatsIndexEnabled(ConnectorSession session)
+ {
+ return session.getProperty(PARTITION_STATS_INDEX_ENABLED, Boolean.class);
+ }
+
+ public static boolean isNoOpIndexEnabled(ConnectorSession session)
+ {
+ return !isRecordLevelIndexEnabled(session) && !isSecondaryIndexEnabled(session) && !isColumnStatsIndexEnabled(session);
+ }
+
+ public static Duration getDynamicFilteringWaitTimeout(ConnectorSession session)
+ {
+ return session.getProperty(DYNAMIC_FILTERING_WAIT_TIMEOUT, Duration.class);
+ }
+
+ public static Duration getColumnStatsWaitTimeout(ConnectorSession session)
+ {
+ return session.getProperty(COLUMN_STATS_WAIT_TIMEOUT, Duration.class);
+ }
+
+ public static Duration getRecordIndexWaitTimeout(ConnectorSession session)
+ {
+ return session.getProperty(RECORD_INDEX_WAIT_TIMEOUT, Duration.class);
+ }
+
+ public static Duration getSecondaryIndexWaitTimeout(ConnectorSession session)
+ {
+ return session.getProperty(SECONDARY_INDEX_WAIT_TIMEOUT, Duration.class);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java
new file mode 100644
index 000000000000..c0c15eada837
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplit.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hudi.file.HudiBaseFile;
+import io.trino.plugin.hudi.file.HudiLogFile;
+import io.trino.spi.SplitWeight;
+import io.trino.spi.connector.ConnectorSplit;
+import io.trino.spi.predicate.TupleDomain;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static io.airlift.slice.SizeOf.estimatedSizeOf;
+import static io.airlift.slice.SizeOf.instanceSize;
+import static java.lang.Math.toIntExact;
+import static java.util.Objects.requireNonNull;
+
+public class HudiSplit
+ implements ConnectorSplit
+{
+ private static final int INSTANCE_SIZE = toIntExact(instanceSize(HudiSplit.class));
+
+ private final Optional<HudiBaseFile> baseFile;
+ private final List<HudiLogFile> logFiles;
+ private final String commitTime;
+ private final TupleDomain<HiveColumnHandle> predicate;
+ private final List<HivePartitionKey> partitionKeys;
+ private final SplitWeight splitWeight;
+
+ @JsonCreator
+ public HudiSplit(
+ @JsonProperty("baseFile") HudiBaseFile baseFile,
+ @JsonProperty("logFiles") List<HudiLogFile> logFiles,
+ @JsonProperty("commitTime") String commitTime,
+ @JsonProperty("predicate") TupleDomain<HiveColumnHandle> predicate,
+ @JsonProperty("partitionKeys") List<HivePartitionKey> partitionKeys,
+ @JsonProperty("splitWeight") SplitWeight splitWeight)
+ {
+ this.baseFile = Optional.ofNullable(baseFile);
+ this.logFiles = requireNonNull(logFiles, "logFiles is null");
+ this.commitTime = requireNonNull(commitTime, "commitTime is null");
+ this.predicate = requireNonNull(predicate, "predicate is null");
+ this.partitionKeys = ImmutableList.copyOf(requireNonNull(partitionKeys, "partitionKeys is null"));
+ this.splitWeight = requireNonNull(splitWeight, "splitWeight is null");
+ }
+
+ @Override
+ public Map<String, String> getSplitInfo()
+ {
+ return ImmutableMap.<String, String>builder()
+ .put("baseFile", baseFile.toString())
+ .put("logFiles", logFiles.toString())
+ .put("commitTime", commitTime)
+ .buildOrThrow();
+ }
+
+ @JsonProperty
+ public Optional<HudiBaseFile> getBaseFile()
+ {
+ return baseFile;
+ }
+
+ @JsonProperty
+ public List<HudiLogFile> getLogFiles()
+ {
+ return logFiles;
+ }
+
+ @JsonProperty
+ public String getCommitTime()
+ {
+ return commitTime;
+ }
+
+ @JsonProperty
+ @Override
+ public SplitWeight getSplitWeight()
+ {
+ return splitWeight;
+ }
+
+ @JsonProperty
+ public TupleDomain<HiveColumnHandle> getPredicate()
+ {
+ return predicate;
+ }
+
+ @JsonProperty
+ public List<HivePartitionKey> getPartitionKeys()
+ {
+ return partitionKeys;
+ }
+
+ @Override
+ public long getRetainedSizeInBytes()
+ {
+ return INSTANCE_SIZE
+ + 10
+ + 10
+ + estimatedSizeOf(commitTime)
+ + splitWeight.getRetainedSizeInBytes()
+ + predicate.getRetainedSizeInBytes(HiveColumnHandle::getRetainedSizeInBytes)
+ + estimatedSizeOf(partitionKeys, HivePartitionKey::estimatedSizeInBytes);
+ }
+
+ @Override
+ public String toString()
+ {
+ return toStringHelper(this)
+ .addValue(baseFile)
+ .addValue(logFiles)
+ .addValue(commitTime)
+ .toString();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java
new file mode 100644
index 000000000000..8f774e3f5f15
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitManager.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import io.airlift.log.Logger;
+import io.trino.metastore.HiveMetastore;
+import io.trino.metastore.Partition;
+import io.trino.metastore.StorageFormat;
+import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitSource;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HiveTransactionHandle;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorSplitManager;
+import io.trino.spi.connector.ConnectorSplitSource;
+import io.trino.spi.connector.ConnectorTableHandle;
+import io.trino.spi.connector.ConnectorTransactionHandle;
+import io.trino.spi.connector.Constraint;
+import io.trino.spi.connector.DynamicFilter;
+import io.trino.spi.connector.TableNotFoundException;
+import io.trino.spi.security.ConnectorIdentity;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+import static io.trino.plugin.hive.metastore.MetastoreUtil.computePartitionKeyFilter;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_PARTITION_NOT_FOUND;
+import static io.trino.plugin.hudi.HudiSessionProperties.getDynamicFilteringWaitTimeout;
+import static io.trino.plugin.hudi.HudiSessionProperties.getMaxOutstandingSplits;
+import static io.trino.plugin.hudi.HudiSessionProperties.getMaxSplitsPerSecond;
+import static io.trino.plugin.hudi.partition.HiveHudiPartitionInfo.NON_PARTITION;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+
+public class HudiSplitManager
+ implements ConnectorSplitManager
+{
+ private static final Logger log = Logger.get(HudiSplitManager.class);
+ private final BiFunction<ConnectorIdentity, HiveTransactionHandle, HiveMetastore> metastoreProvider;
+ private final ExecutorService executor;
+ private final ScheduledExecutorService splitLoaderExecutorService;
+
+ @Inject
+ public HudiSplitManager(
+ BiFunction<ConnectorIdentity, HiveTransactionHandle, HiveMetastore> metastoreProvider,
+ @ForHudiSplitManager ExecutorService executor,
+ @ForHudiSplitSource ScheduledExecutorService splitLoaderExecutorService)
+ {
+ this.metastoreProvider = requireNonNull(metastoreProvider, "metastoreProvider is null");
+ this.executor = requireNonNull(executor, "executor is null");
+ this.splitLoaderExecutorService = requireNonNull(splitLoaderExecutorService, "splitLoaderExecutorService is null");
+ }
+
+ @Override
+ public ConnectorSplitSource getSplits(
+ ConnectorTransactionHandle transaction,
+ ConnectorSession session,
+ ConnectorTableHandle tableHandle,
+ DynamicFilter dynamicFilter,
+ Constraint constraint)
+ {
+ HudiTableHandle hudiTableHandle = (HudiTableHandle) tableHandle;
+ HiveMetastore metastore = metastoreProvider.apply(session.getIdentity(), (HiveTransactionHandle) transaction);
+ Lazy<Map<String, Partition>> lazyAllPartitions = Lazy.lazily(() -> {
+ HoodieTimer timer = HoodieTimer.start();
+ Map<String, Partition> allPartitions = getPartitions(metastore, hudiTableHandle);
+ log.info("Found %s partitions for table %s.%s in %s ms",
+ allPartitions.size(), hudiTableHandle.getSchemaName(), hudiTableHandle.getTableName(), timer.endTimer());
+ return allPartitions;
+ });
+
+ HudiSplitSource splitSource = new HudiSplitSource(
+ session,
+ hudiTableHandle,
+ executor,
+ splitLoaderExecutorService,
+ getMaxSplitsPerSecond(session),
+ getMaxOutstandingSplits(session),
+ lazyAllPartitions,
+ dynamicFilter,
+ getDynamicFilteringWaitTimeout(session));
+ return new ClassLoaderSafeConnectorSplitSource(splitSource, HudiSplitManager.class.getClassLoader());
+ }
+
+ private static Map<String, Partition> getPartitions(
+ HiveMetastore metastore,
+ HudiTableHandle tableHandle)
+ {
+ List<HiveColumnHandle> partitionColumns = tableHandle.getPartitionColumns();
+ if (partitionColumns.isEmpty()) {
+ return ImmutableMap.of(
+ NON_PARTITION, Partition.builder()
+ .setDatabaseName(tableHandle.getSchemaName())
+ .setTableName(tableHandle.getTableName())
+ .withStorage(storageBuilder ->
+ storageBuilder.setLocation(tableHandle.getBasePath())
+ .setStorageFormat(StorageFormat.NULL_STORAGE_FORMAT))
+ .setColumns(ImmutableList.of())
+ .setValues(ImmutableList.of())
+ .build());
+ }
+
+ List<String> partitionNames = metastore.getPartitionNamesByFilter(
+ tableHandle.getSchemaName(),
+ tableHandle.getTableName(),
+ partitionColumns.stream().map(HiveColumnHandle::getName).collect(Collectors.toList()),
+ computePartitionKeyFilter(partitionColumns, tableHandle.getPartitionPredicates()))
+ .orElseThrow(() -> new TableNotFoundException(tableHandle.getSchemaTableName()));
+ Map<String, Optional<Partition>> partitionsByNames = metastore.getPartitionsByNames(tableHandle.getTable(), partitionNames);
+ List<String> partitionsNotFound = partitionsByNames.entrySet().stream().filter(e -> e.getValue().isEmpty()).map(Map.Entry::getKey).toList();
+ if (!partitionsNotFound.isEmpty()) {
+ throw new TrinoException(HUDI_PARTITION_NOT_FOUND, format("Cannot find partitions in metastore: %s", partitionsNotFound));
+ }
+ return partitionsByNames
+ .entrySet().stream()
+ .filter(e -> e.getValue().isPresent())
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get()));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java
new file mode 100644
index 000000000000..f0286aea7e95
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiSplitSource.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+import io.airlift.concurrent.BoundedExecutor;
+import io.airlift.units.DataSize;
+import io.airlift.units.Duration;
+import io.trino.metastore.Partition;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hive.util.AsyncQueue;
+import io.trino.plugin.hive.util.HiveUtil;
+import io.trino.plugin.hive.util.ThrottledAsyncQueue;
+import io.trino.plugin.hudi.query.HudiDirectoryLister;
+import io.trino.plugin.hudi.query.HudiSnapshotDirectoryLister;
+import io.trino.plugin.hudi.split.HudiBackgroundSplitLoader;
+import io.trino.plugin.hudi.split.HudiSplitWeightProvider;
+import io.trino.plugin.hudi.split.SizeBasedSplitWeightProvider;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorSplit;
+import io.trino.spi.connector.ConnectorSplitSource;
+import io.trino.spi.connector.DynamicFilter;
+import io.trino.spi.predicate.Domain;
+import io.trino.spi.predicate.NullableValue;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.util.Lazy;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.OptionalInt;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+import static io.airlift.concurrent.MoreFutures.toCompletableFuture;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CANNOT_OPEN_SPLIT;
+import static io.trino.plugin.hudi.HudiSessionProperties.getMinimumAssignedSplitWeight;
+import static io.trino.plugin.hudi.HudiSessionProperties.getSplitGeneratorParallelism;
+import static io.trino.plugin.hudi.HudiSessionProperties.getStandardSplitWeightSize;
+import static io.trino.plugin.hudi.HudiSessionProperties.isHudiMetadataTableEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isSizeBasedSplitWeightsEnabled;
+import static java.util.Objects.requireNonNull;
+import static java.util.concurrent.CompletableFuture.completedFuture;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public class HudiSplitSource
+ implements ConnectorSplitSource
+{
+ private static final ConnectorSplitBatch EMPTY_BATCH = new ConnectorSplitBatch(ImmutableList.of(), false);
+ private final AsyncQueue<ConnectorSplit> queue;
+ private final ScheduledFuture splitLoaderFuture;
+ private final AtomicReference<TrinoException> trinoException = new AtomicReference<>();
+ private final DynamicFilter dynamicFilter;
+ private final long dynamicFilteringWaitTimeoutMillis;
+ private final Stopwatch dynamicFilterWaitStopwatch;
+
+ public HudiSplitSource(
+ ConnectorSession session,
+ HudiTableHandle tableHandle,
+ ExecutorService executor,
+ ScheduledExecutorService splitLoaderExecutorService,
+ int maxSplitsPerSecond,
+ int maxOutstandingSplits,
+ Lazy<Map<String, Partition>> lazyPartitions,
+ DynamicFilter dynamicFilter,
+ Duration dynamicFilteringWaitTimeoutMillis)
+ {
+ boolean enableMetadataTable = isHudiMetadataTableEnabled(session);
+ Lazy<HoodieTableMetadata> lazyTableMetadata = Lazy.lazily(() -> {
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
+ .enable(enableMetadataTable)
+ .build();
+ HoodieTableMetaClient metaClient = tableHandle.getMetaClient();
+ HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorage().getConf());
+ return HoodieTableMetadata.create(
+ engineContext,
+ tableHandle.getMetaClient().getStorage(), metadataConfig, metaClient.getBasePath().toString(), true);
+ });
+
+ HudiDirectoryLister hudiDirectoryLister = new HudiSnapshotDirectoryLister(
+ session,
+ tableHandle,
+ enableMetadataTable,
+ lazyTableMetadata,
+ lazyPartitions);
+
+ this.queue = new ThrottledAsyncQueue<>(maxSplitsPerSecond, maxOutstandingSplits, executor);
+ HudiBackgroundSplitLoader splitLoader = new HudiBackgroundSplitLoader(
+ session,
+ tableHandle,
+ hudiDirectoryLister,
+ queue,
+ new BoundedExecutor(executor, getSplitGeneratorParallelism(session)),
+ createSplitWeightProvider(session),
+ lazyPartitions,
+ enableMetadataTable,
+ lazyTableMetadata,
+ throwable -> {
+ trinoException.compareAndSet(null, new TrinoException(HUDI_CANNOT_OPEN_SPLIT,
+ "Failed to generate splits for " + tableHandle.getSchemaTableName(), throwable));
+ queue.finish();
+ });
+ this.splitLoaderFuture = splitLoaderExecutorService.schedule(splitLoader, 0, TimeUnit.MILLISECONDS);
+ this.dynamicFilter = requireNonNull(dynamicFilter, "dynamicFilter is null");
+ this.dynamicFilteringWaitTimeoutMillis = dynamicFilteringWaitTimeoutMillis.toMillis();
+ this.dynamicFilterWaitStopwatch = Stopwatch.createStarted();
+ }
+
+ @Override
+ public CompletableFuture<ConnectorSplitBatch> getNextBatch(int maxSize)
+ {
+ // If dynamic filtering is enabled and we haven't timed out, wait for the build side to provide the dynamic filter.
+ long timeLeft = dynamicFilteringWaitTimeoutMillis - dynamicFilterWaitStopwatch.elapsed(MILLISECONDS);
+ if (dynamicFilter.isAwaitable() && timeLeft > 0) {
+ // If the filter is not ready, return an empty batch. The query engine will call getNextBatch() again.
+ // As long as isFinished() is false, effectively polling until the filter is ready or timeout occurs.
+ return dynamicFilter.isBlocked()
+ .thenApply(_ -> EMPTY_BATCH)
+ .completeOnTimeout(EMPTY_BATCH, timeLeft, MILLISECONDS);
+ }
+
+ TupleDomain<HiveColumnHandle> dynamicFilterPredicate =
+ dynamicFilter.getCurrentPredicate().transformKeys(HiveColumnHandle.class::cast);
+
+ if (dynamicFilterPredicate.isNone()) {
+ close();
+ return completedFuture(new ConnectorSplitBatch(ImmutableList.of(), true));
+ }
+
+ boolean noMoreSplits = isFinished();
+ Throwable throwable = trinoException.get();
+ if (throwable != null) {
+ return CompletableFuture.failedFuture(throwable);
+ }
+
+ return toCompletableFuture(Futures.transform(
+ queue.getBatchAsync(maxSize),
+ splits ->
+ {
+ List<ConnectorSplit> filteredSplits = splits.stream()
+ .filter(split -> partitionMatchesPredicate((HudiSplit) split, dynamicFilterPredicate))
+ .collect(toImmutableList());
+ return new ConnectorSplitBatch(filteredSplits, noMoreSplits);
+ },
+ directExecutor()));
+ }
+
+ @Override
+ public void close()
+ {
+ queue.finish();
+ }
+
+ @Override
+ public boolean isFinished()
+ {
+ return splitLoaderFuture.isDone() && queue.isFinished();
+ }
+
+ public static HudiSplitWeightProvider createSplitWeightProvider(ConnectorSession session)
+ {
+ if (isSizeBasedSplitWeightsEnabled(session)) {
+ DataSize standardSplitWeightSize = getStandardSplitWeightSize(session);
+ double minimumAssignedSplitWeight = getMinimumAssignedSplitWeight(session);
+ return new SizeBasedSplitWeightProvider(minimumAssignedSplitWeight, standardSplitWeightSize);
+ }
+ return HudiSplitWeightProvider.uniformStandardWeightProvider();
+ }
+
+ static boolean partitionMatchesPredicate(
+ HudiSplit split,
+ TupleDomain<HiveColumnHandle> dynamicFilterPredicate)
+ {
+ if (dynamicFilterPredicate.isNone()) {
+ return false;
+ }
+
+ // Pre-process the filter predicate to get a map of relevant partition domains keyed by partition column name
+ Map<String, Map.Entry<HiveColumnHandle, Domain>> filterPartitionDomains = new HashMap<>();
+ if (dynamicFilterPredicate.getDomains().isPresent()) {
+ for (Map.Entry<HiveColumnHandle, Domain> entry : dynamicFilterPredicate.getDomains().get().entrySet()) {
+ HiveColumnHandle column = entry.getKey();
+ if (column.isPartitionKey()) {
+ filterPartitionDomains.put(column.getName(), entry);
+ }
+ }
+ }
+
+ // Match each partition key from the split against the pre-processed filter domains
+ for (HivePartitionKey splitPartitionKey : split.getPartitionKeys()) {
+ Map.Entry<HiveColumnHandle, Domain> filterInfo = filterPartitionDomains.get(splitPartitionKey.name());
+
+ if (filterInfo == null) {
+ // filterInfo is null, the partition key is not constrained by the filter
+ continue;
+ }
+
+ HiveColumnHandle filterColumnHandle = filterInfo.getKey();
+ Domain filterDomain = filterInfo.getValue();
+
+ NullableValue value = HiveUtil.getPrefilledColumnValue(
+ filterColumnHandle,
+ splitPartitionKey,
+ null, OptionalInt.empty(), 0, 0, "");
+
+ // Split does not match this filter condition
+ if (!filterDomain.includesNullableValue(value.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java
new file mode 100644
index 000000000000..d6a152446ef0
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableHandle.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableSet;
+import io.trino.metastore.Table;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorTableHandle;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static io.trino.spi.connector.SchemaTableName.schemaTableName;
+import static java.util.Objects.requireNonNull;
+
+public class HudiTableHandle
+ implements ConnectorTableHandle
+{
+ private final String schemaName;
+ private final String tableName;
+ private final String basePath;
+ private final HoodieTableType tableType;
+ private final List<HiveColumnHandle> partitionColumns;
+ // Used only for validation when config property hudi.query-partition-filter-required is enabled
+ private final Set<HiveColumnHandle> constraintColumns;
+ private final TupleDomain<HiveColumnHandle> partitionPredicates;
+ private final TupleDomain<HiveColumnHandle> regularPredicates;
+ // Coordinator-only
+ private final transient Optional<Table> table;
+ private final transient Optional<Lazy<HoodieTableMetaClient>> lazyMetaClient;
+ private final transient Optional<Lazy<String>> lazyLatestCommitTime;
+
+ @JsonCreator
+ public HudiTableHandle(
+ @JsonProperty("schemaName") String schemaName,
+ @JsonProperty("tableName") String tableName,
+ @JsonProperty("basePath") String basePath,
+ @JsonProperty("tableType") HoodieTableType tableType,
+ @JsonProperty("partitionColumns") List<HiveColumnHandle> partitionColumns,
+ @JsonProperty("partitionPredicates") TupleDomain<HiveColumnHandle> partitionPredicates,
+ @JsonProperty("regularPredicates") TupleDomain<HiveColumnHandle> regularPredicates)
+ {
+ this(Optional.empty(), Optional.empty(), schemaName, tableName, basePath, tableType, partitionColumns, ImmutableSet.of(), partitionPredicates, regularPredicates);
+ }
+
+ public HudiTableHandle(
+ Optional<Table> table,
+ Optional<Lazy<HoodieTableMetaClient>> lazyMetaClient,
+ String schemaName,
+ String tableName,
+ String basePath,
+ HoodieTableType tableType,
+ List<HiveColumnHandle> partitionColumns,
+ Set<HiveColumnHandle> constraintColumns,
+ TupleDomain<HiveColumnHandle> partitionPredicates,
+ TupleDomain<HiveColumnHandle> regularPredicates)
+ {
+ this.table = requireNonNull(table, "table is null");
+ this.lazyMetaClient = requireNonNull(lazyMetaClient, "lazyMetaClient is null");
+ this.lazyLatestCommitTime = Optional.of(Lazy.lazily(() ->
+ getMetaClient().getActiveTimeline()
+ .getCommitsTimeline()
+ .filterCompletedInstants()
+ .lastInstant()
+ .map(HoodieInstant::requestedTime)
+ .orElseThrow(() -> new TrinoException(HudiErrorCode.HUDI_NO_VALID_COMMIT, "Table has no valid commits"))));
+ this.schemaName = requireNonNull(schemaName, "schemaName is null");
+ this.tableName = requireNonNull(tableName, "tableName is null");
+ this.basePath = requireNonNull(basePath, "basePath is null");
+ this.tableType = requireNonNull(tableType, "tableType is null");
+ this.partitionColumns = requireNonNull(partitionColumns, "partitionColumns is null");
+ this.constraintColumns = requireNonNull(constraintColumns, "constraintColumns is null");
+ this.partitionPredicates = requireNonNull(partitionPredicates, "partitionPredicates is null");
+ this.regularPredicates = requireNonNull(regularPredicates, "regularPredicates is null");
+ }
+
+ public Table getTable()
+ {
+ checkArgument(table.isPresent(),
+ "getTable() called on a table handle that has no metastore table object; "
+ + "this is likely because it is called on the worker.");
+ return table.get();
+ }
+
+ public HoodieTableMetaClient getMetaClient()
+ {
+ checkArgument(lazyMetaClient.isPresent(),
+ "getMetaClient() called on a table handle that has no Hudi meta-client; "
+ + "this is likely because it is called on the worker.");
+ return lazyMetaClient.get().get();
+ }
+
+ public String getLatestCommitTime()
+ {
+ checkArgument(lazyLatestCommitTime.isPresent(),
+ "getLatestCommitTime() called on a table handle that has no Hudi meta-client; "
+ + "this is likely because it is called on the worker.");
+ return lazyLatestCommitTime.get().get();
+ }
+
+ @JsonProperty
+ public String getSchemaName()
+ {
+ return schemaName;
+ }
+
+ @JsonProperty
+ public String getTableName()
+ {
+ return tableName;
+ }
+
+ @JsonProperty
+ public String getBasePath()
+ {
+ return basePath;
+ }
+
+ @JsonProperty
+ public HoodieTableType getTableType()
+ {
+ return tableType;
+ }
+
+ @JsonProperty
+ public TupleDomain<HiveColumnHandle> getPartitionPredicates()
+ {
+ return partitionPredicates;
+ }
+
+ @JsonProperty
+ public List<HiveColumnHandle> getPartitionColumns()
+ {
+ return partitionColumns;
+ }
+
+ // do not serialize constraint columns as they are not needed on workers
+ @JsonIgnore
+ public Set<HiveColumnHandle> getConstraintColumns()
+ {
+ return constraintColumns;
+ }
+
+ @JsonProperty
+ public TupleDomain<HiveColumnHandle> getRegularPredicates()
+ {
+ return regularPredicates;
+ }
+
+ public SchemaTableName getSchemaTableName()
+ {
+ return schemaTableName(schemaName, tableName);
+ }
+
+ HudiTableHandle applyPredicates(
+ Set<HiveColumnHandle> constraintColumns,
+ TupleDomain<HiveColumnHandle> partitionTupleDomain,
+ TupleDomain<HiveColumnHandle> regularTupleDomain)
+ {
+ return new HudiTableHandle(
+ table,
+ lazyMetaClient,
+ schemaName,
+ tableName,
+ basePath,
+ tableType,
+ partitionColumns,
+ constraintColumns,
+ partitionPredicates.intersect(partitionTupleDomain),
+ regularPredicates.intersect(regularTupleDomain));
+ }
+
+ @Override
+ public String toString()
+ {
+ return getSchemaTableName().toString();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java
new file mode 100644
index 000000000000..1a7d6984e16f
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableInfo.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import io.trino.spi.connector.SchemaTableName;
+
+import static java.util.Objects.requireNonNull;
+
+public record HudiTableInfo(SchemaTableName table, String tableType, String basePath)
+{
+ public HudiTableInfo
+ {
+ requireNonNull(table, "table is null");
+ requireNonNull(tableType, "tableType is null");
+ requireNonNull(basePath, "basePath is null");
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableName.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableName.java
new file mode 100644
index 000000000000..a16f35e85ceb
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableName.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Verify.verify;
+import static io.trino.plugin.hudi.TableType.DATA;
+import static java.util.Locale.ENGLISH;
+import static java.util.Objects.requireNonNull;
+
+public record HudiTableName(String tableName, TableType tableType)
+{
+ private static final Pattern TABLE_PATTERN = Pattern.compile(
+ "(?<table>[^$@]+)(?:\\$(?<type>(?i:timeline)))?");
+
+ public HudiTableName(String tableName, TableType tableType)
+ {
+ this.tableName = requireNonNull(tableName, "tableName is null");
+ this.tableType = requireNonNull(tableType, "tableType is null");
+ }
+
+ public String tableNameWithType()
+ {
+ return tableName + "$" + tableType.name().toLowerCase(ENGLISH);
+ }
+
+ @Override
+ public String toString()
+ {
+ return tableNameWithType();
+ }
+
+ public static Optional<HudiTableName> from(String name)
+ {
+ Matcher match = TABLE_PATTERN.matcher(name);
+ if (!match.matches()) {
+ return Optional.empty();
+ }
+
+ String table = match.group("table");
+ String typeString = match.group("type");
+
+ TableType type = DATA;
+ if (typeString != null) {
+ type = TableType.valueOf(typeString.toUpperCase(ENGLISH));
+ verify(type != DATA, "parsedType is unexpectedly DATA");
+ }
+
+ return Optional.of(new HudiTableName(table, type));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java
new file mode 100644
index 000000000000..ddb2c01e8bc2
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTableProperties.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import io.trino.spi.session.PropertyMetadata;
+import io.trino.spi.type.ArrayType;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.trino.spi.session.PropertyMetadata.stringProperty;
+import static io.trino.spi.type.VarcharType.VARCHAR;
+import static java.util.Locale.ENGLISH;
+
+public class HudiTableProperties
+{
+ public static final String LOCATION_PROPERTY = "location";
+ public static final String PARTITIONED_BY_PROPERTY = "partitioned_by";
+
+ private final List<PropertyMetadata<?>> tableProperties;
+
+ @Inject
+ public HudiTableProperties()
+ {
+ tableProperties = ImmutableList.<PropertyMetadata<?>>builder()
+ .add(stringProperty(
+ LOCATION_PROPERTY,
+ "File system location URI for the table",
+ null,
+ false))
+ .add(new PropertyMetadata<>(
+ PARTITIONED_BY_PROPERTY,
+ "Partition columns",
+ new ArrayType(VARCHAR),
+ List.class,
+ ImmutableList.of(),
+ false,
+ value -> ((Collection<String>) value).stream()
+ .map(name -> name.toLowerCase(ENGLISH))
+ .collect(toImmutableList()),
+ value -> value))
+ .build();
+ }
+
+ public List<PropertyMetadata<?>> getTableProperties()
+ {
+ return tableProperties;
+ }
+
+ public static Optional<String> getTableLocation(Map<String, Object> tableProperties)
+ {
+ return Optional.ofNullable((String) tableProperties.get(LOCATION_PROPERTY));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java
new file mode 100644
index 000000000000..994a4d929359
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiTransactionManager.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.errorprone.annotations.concurrent.GuardedBy;
+import com.google.inject.Inject;
+import io.trino.spi.classloader.ThreadContextClassLoader;
+import io.trino.spi.connector.ConnectorTransactionHandle;
+import io.trino.spi.security.ConnectorIdentity;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.Objects.requireNonNull;
+
+public class HudiTransactionManager
+{
+ private final Map<ConnectorTransactionHandle, MemoizedMetadata> transactions = new ConcurrentHashMap<>();
+ private final HudiMetadataFactory metadataFactory;
+
+ @Inject
+ public HudiTransactionManager(HudiMetadataFactory metadataFactory)
+ {
+ this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null");
+ }
+
+ public HudiMetadata get(ConnectorTransactionHandle transaction, ConnectorIdentity identity)
+ {
+ HudiMetadata metadata = transactions.get(transaction).get(identity);
+ checkArgument(metadata != null, "no such transaction: %s", transaction);
+ return metadata;
+ }
+
+ public void commit(ConnectorTransactionHandle transaction)
+ {
+ MemoizedMetadata metadata = transactions.remove(transaction);
+ checkArgument(metadata != null, "no such transaction: %s", transaction);
+ }
+
+ public void rollback(ConnectorTransactionHandle transaction)
+ {
+ MemoizedMetadata transactionalMetadata = transactions.remove(transaction);
+ checkArgument(transactionalMetadata != null, "no such transaction: %s", transaction);
+ }
+
+ public void put(ConnectorTransactionHandle transaction)
+ {
+ MemoizedMetadata existing = transactions.putIfAbsent(transaction, new MemoizedMetadata());
+ checkState(existing == null, "transaction already exists: %s", existing);
+ }
+
+ private class MemoizedMetadata
+ {
+ @GuardedBy("this")
+ private HudiMetadata metadata;
+
+ public synchronized HudiMetadata get(ConnectorIdentity identity)
+ {
+ if (metadata == null) {
+ try (ThreadContextClassLoader _ = new ThreadContextClassLoader(getClass().getClassLoader())) {
+ metadata = metadataFactory.create(identity);
+ }
+ }
+ return metadata;
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java
new file mode 100644
index 000000000000..05c85efdbe24
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/HudiUtil.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import io.trino.filesystem.FileIterator;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.metastore.HivePartition;
+import io.trino.metastore.HiveType;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hive.HivePartitionManager;
+import io.trino.plugin.hive.avro.AvroHiveFileUtils;
+import io.trino.plugin.hudi.storage.HudiTrinoStorage;
+import io.trino.plugin.hudi.storage.TrinoStorageConfiguration;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.Domain;
+import io.trino.spi.predicate.NullableValue;
+import io.trino.spi.predicate.TupleDomain;
+import io.trino.spi.type.VarcharType;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.exception.TableNotFoundException;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.StoragePath;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA;
+import static io.trino.plugin.hive.util.HiveUtil.checkCondition;
+import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMNS;
+import static io.trino.plugin.hive.util.SerdeConstants.LIST_COLUMN_TYPES;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_BAD_DATA;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_FILESYSTEM_ERROR;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_META_CLIENT_ERROR;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_UNSUPPORTED_FILE_FORMAT;
+import static org.apache.hudi.common.model.HoodieRecord.HOODIE_META_COLUMNS;
+
+public final class HudiUtil
+{
+ private HudiUtil() {}
+
+ public static HoodieFileFormat getHudiFileFormat(String path)
+ {
+ String extension = getFileExtension(path);
+ if (extension.equals(HoodieFileFormat.PARQUET.getFileExtension())) {
+ return HoodieFileFormat.PARQUET;
+ }
+ if (extension.equals(HoodieFileFormat.HOODIE_LOG.getFileExtension())) {
+ return HoodieFileFormat.HOODIE_LOG;
+ }
+ if (extension.equals(HoodieFileFormat.ORC.getFileExtension())) {
+ return HoodieFileFormat.ORC;
+ }
+ if (extension.equals(HoodieFileFormat.HFILE.getFileExtension())) {
+ return HoodieFileFormat.HFILE;
+ }
+ throw new TrinoException(HUDI_UNSUPPORTED_FILE_FORMAT, "Hoodie InputFormat not implemented for base file of type " + extension);
+ }
+
+ private static String getFileExtension(String fullName)
+ {
+ String fileName = Location.of(fullName).fileName();
+ int dotIndex = fileName.lastIndexOf('.');
+ return dotIndex == -1 ? "" : fileName.substring(dotIndex);
+ }
+
+ public static boolean hudiMetadataExists(TrinoFileSystem trinoFileSystem, Location baseLocation)
+ {
+ try {
+ Location metaLocation = baseLocation.appendPath(HoodieTableMetaClient.METAFOLDER_NAME);
+ FileIterator iterator = trinoFileSystem.listFiles(metaLocation);
+ // If there is at least one file in the .hoodie directory, it's a valid Hudi table
+ return iterator.hasNext();
+ }
+ catch (IOException e) {
+ throw new TrinoException(HUDI_FILESYSTEM_ERROR, "Failed to check for Hudi table at location: " + baseLocation, e);
+ }
+ }
+
+ public static boolean partitionMatchesPredicates(
+ SchemaTableName tableName,
+ String hivePartitionName,
+ List<HiveColumnHandle> partitionColumnHandles,
+ TupleDomain<HiveColumnHandle> constraintSummary)
+ {
+ HivePartition partition = HivePartitionManager.parsePartition(
+ tableName, hivePartitionName, partitionColumnHandles);
+
+ return partitionMatches(partitionColumnHandles, constraintSummary, partition);
+ }
+
+ public static boolean partitionMatches(List<HiveColumnHandle> partitionColumns, TupleDomain<HiveColumnHandle> constraintSummary, HivePartition partition)
+ {
+ if (constraintSummary.isNone()) {
+ return false;
+ }
+ Map<HiveColumnHandle, Domain> domains = constraintSummary.getDomains().orElseGet(ImmutableMap::of);
+ for (HiveColumnHandle column : partitionColumns) {
+ NullableValue value = partition.getKeys().get(column);
+ Domain allowedDomain = domains.get(column);
+ if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static List<HivePartitionKey> buildPartitionKeys(List<HiveColumnHandle> keys, List<String> values)
+ {
+ checkCondition(keys.size() == values.size(), HIVE_INVALID_METADATA,
+ "Expected %s partition key values, but got %s. Keys: %s, Values: %s.",
+ keys.size(), values.size(), keys, values);
+ ImmutableList.Builder<HivePartitionKey> partitionKeys = ImmutableList.builder();
+ for (int i = 0; i < keys.size(); i++) {
+ String name = keys.get(i).getName();
+ String value = values.get(i);
+ partitionKeys.add(new HivePartitionKey(name, value));
+ }
+ return partitionKeys.build();
+ }
+
+ public static HoodieTableMetaClient buildTableMetaClient(
+ TrinoFileSystem fileSystem,
+ String tableName,
+ String basePath)
+ {
+ try {
+ return HoodieTableMetaClient.builder()
+ .setStorage(new HudiTrinoStorage(fileSystem, new TrinoStorageConfiguration()))
+ .setBasePath(basePath)
+ .build();
+ }
+ catch (TableNotFoundException e) {
+ throw new TrinoException(HUDI_BAD_DATA,
+ "Location of table %s does not contain Hudi table metadata: %s".formatted(tableName, basePath));
+ }
+ catch (Throwable e) {
+ throw new TrinoException(HUDI_META_CLIENT_ERROR,
+ "Unable to load Hudi meta client for table %s (%s)".formatted(tableName, basePath));
+ }
+ }
+
+ public static Schema constructSchema(List<String> columnNames, List<HiveType> columnTypes)
+ {
+ // Convert lists into the format expected by the utility class
+ String columnNamesString = String.join(",", columnNames);
+ String columnTypesString = columnTypes.stream()
+ .map(HiveType::getHiveTypeName)
+ .map(Object::toString)
+ .collect(Collectors.joining(":"));
+
+ // Create the properties map
+ Map<String, String> properties = new HashMap<>();
+ properties.put(LIST_COLUMNS, columnNamesString);
+ properties.put(LIST_COLUMN_TYPES, columnTypesString);
+
+ // Call the public static method to build the schema
+ try {
+ // Pass null for the file system as we are not reading from a URL
+ return AvroHiveFileUtils.determineSchemaOrThrowException(null, properties);
+ }
+ catch (IOException e) {
+ // The IOException is declared on the method, but this path shouldn't throw it
+ throw new UncheckedIOException("Failed to construct Avro schema", e);
+ }
+ }
+
+ public static Schema constructSchema(Schema dataSchema, List<String> columnNames)
+ {
+ SchemaBuilder.RecordBuilder<Schema> schemaBuilder = SchemaBuilder.record("baseRecord");
+ SchemaBuilder.FieldAssembler<Schema> fieldBuilder = schemaBuilder.fields();
+ for (String columnName : columnNames) {
+ Schema originalFieldSchema = dataSchema.getField(columnName).schema();
+ Schema typeForNewField;
+
+ // Check if the original field schema is already nullable (i.e., a UNION containing NULL)
+ if (originalFieldSchema.isNullable()) {
+ typeForNewField = originalFieldSchema;
+ }
+ else {
+ typeForNewField = Schema.createUnion(Schema.create(Schema.Type.NULL), originalFieldSchema);
+ }
+
+ fieldBuilder = fieldBuilder
+ .name(columnName)
+ .type(typeForNewField)
+ .withDefault(null);
+ }
+ return fieldBuilder.endRecord();
+ }
+
+ public static List<HiveColumnHandle> prependHudiMetaColumns(List<HiveColumnHandle> dataColumns)
+ {
+ //For efficient lookup
+ Set<String> dataColumnNames = dataColumns.stream()
+ .map(HiveColumnHandle::getName)
+ .collect(Collectors.toSet());
+
+ // If all Hudi meta columns are already present, return the original list
+ if (dataColumnNames.containsAll(HOODIE_META_COLUMNS)) {
+ return dataColumns;
+ }
+
+ // Identify only the meta columns that are missing from dataColumns to avoid duplicates
+ List<String> missingMetaColumns = HOODIE_META_COLUMNS.stream()
+ .filter(metaColumn -> !dataColumnNames.contains(metaColumn))
+ .toList();
+
+ List<HiveColumnHandle> columns = new ArrayList<>();
+
+ // Create and prepend the new HiveColumnHandles for the missing meta columns
+ columns.addAll(IntStream.range(0, missingMetaColumns.size())
+ .boxed()
+ .map(i -> new HiveColumnHandle(
+ missingMetaColumns.get(i),
+ i,
+ HiveType.HIVE_STRING,
+ VarcharType.VARCHAR,
+ Optional.empty(),
+ HiveColumnHandle.ColumnType.REGULAR,
+ Optional.empty()))
+ .toList());
+
+ // Add all the original data columns after the new meta columns
+ columns.addAll(dataColumns);
+
+ return columns;
+ }
+
+ public static FileSlice convertToFileSlice(HudiSplit split, String basePath)
+ {
+ String dataFilePath = split.getBaseFile().isPresent()
+ ? split.getBaseFile().get().getPath()
+ : split.getLogFiles().getFirst().getPath();
+ String fileId = FSUtils.getFileIdFromFileName(new StoragePath(dataFilePath).getName());
+ HoodieBaseFile baseFile = split.getBaseFile().isPresent()
+ ? new HoodieBaseFile(dataFilePath, fileId, split.getCommitTime(), null)
+ : null;
+
+ return new FileSlice(
+ new HoodieFileGroupId(FSUtils.getRelativePartitionPath(new StoragePath(basePath), new StoragePath(dataFilePath)), fileId),
+ split.getCommitTime(),
+ baseFile,
+ split.getLogFiles().stream().map(lf -> new HoodieLogFile(lf.getPath())).toList());
+ }
+
+ public static HoodieTableFileSystemView getFileSystemView(
+ HoodieTableMetadata tableMetadata,
+ HoodieTableMetaClient metaClient)
+ {
+ return new HoodieTableFileSystemView(
+ tableMetadata, metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants());
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TableType.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TableType.java
new file mode 100644
index 000000000000..2a8238c77f87
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TableType.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+public enum TableType
+{
+ DATA,
+ TIMELINE
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TimelineTable.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TimelineTable.java
new file mode 100644
index 000000000000..1024d87a0b5c
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/TimelineTable.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi;
+
+import com.google.common.collect.ImmutableList;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.metastore.Table;
+import io.trino.spi.connector.ColumnMetadata;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorTableMetadata;
+import io.trino.spi.connector.ConnectorTransactionHandle;
+import io.trino.spi.connector.InMemoryRecordSet;
+import io.trino.spi.connector.RecordCursor;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.connector.SystemTable;
+import io.trino.spi.predicate.TupleDomain;
+import io.trino.spi.type.Type;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.trino.plugin.hudi.HudiUtil.buildTableMetaClient;
+import static io.trino.spi.type.VarcharType.VARCHAR;
+import static java.util.Objects.requireNonNull;
+
+public class TimelineTable
+ implements SystemTable
+{
+ private final ConnectorTableMetadata tableMetadata;
+ private final List<Type> types;
+ private final TrinoFileSystem fileSystem;
+ private final String location;
+
+ public TimelineTable(TrinoFileSystem fileSystem, SchemaTableName tableName, Table hudiTable)
+ {
+ this.tableMetadata = new ConnectorTableMetadata(requireNonNull(tableName, "tableName is null"),
+ ImmutableList.<ColumnMetadata>builder()
+ .add(new ColumnMetadata("timestamp", VARCHAR))
+ .add(new ColumnMetadata("action", VARCHAR))
+ .add(new ColumnMetadata("state", VARCHAR))
+ .build());
+ this.types = tableMetadata.getColumns().stream().map(ColumnMetadata::getType).collect(toImmutableList());
+ this.fileSystem = requireNonNull(fileSystem, "fileSystem is null");
+ this.location = requireNonNull(hudiTable.getStorage().getLocation(), "location is null");
+ }
+
+ @Override
+ public Distribution getDistribution()
+ {
+ return Distribution.SINGLE_COORDINATOR;
+ }
+
+ @Override
+ public ConnectorTableMetadata getTableMetadata()
+ {
+ return tableMetadata;
+ }
+
+ @Override
+ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint)
+ {
+ HoodieTableMetaClient metaClient = buildTableMetaClient(fileSystem, tableMetadata.getTable().toString(), location);
+ Iterable<List<Object>> records = () -> metaClient.getCommitsTimeline().getInstants().stream()
+ .map(this::getRecord).iterator();
+ return new InMemoryRecordSet(types, records).cursor();
+ }
+
+ private List<Object> getRecord(HoodieInstant hudiInstant)
+ {
+ List<Object> columns = new ArrayList<>();
+ columns.add(hudiInstant.requestedTime());
+ columns.add(hudiInstant.getAction());
+ columns.add(hudiInstant.getState().toString());
+ checkArgument(columns.size() == types.size(), "Expected %s types in row, but got %s values", types.size(), columns.size());
+ return columns;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java
new file mode 100644
index 000000000000..f21a9b56bd01
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiBaseFile.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.file;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hudi.common.model.HoodieBaseFile;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+public class HudiBaseFile
+ implements HudiFile
+{
+ private final String path;
+ private final String fileName;
+ private final long fileSize;
+ private final long modificationTime;
+ private final long start;
+ private final long length;
+
+ public static HudiBaseFile of(HoodieBaseFile baseFile)
+ {
+ return of(baseFile, 0, baseFile.getFileSize());
+ }
+
+ public static HudiBaseFile of(HoodieBaseFile baseFile, long start, long length)
+ {
+ return new HudiBaseFile(baseFile, start, length);
+ }
+
+ @JsonCreator
+ public HudiBaseFile(@JsonProperty("path") String path,
+ @JsonProperty("fileName") String fileName,
+ @JsonProperty("fileSize") long fileSize,
+ @JsonProperty("modificationTime") long modificationTime,
+ @JsonProperty("start") long start,
+ @JsonProperty("length") long length)
+ {
+ this.path = path;
+ this.fileName = fileName;
+ this.fileSize = fileSize;
+ this.modificationTime = modificationTime;
+ this.start = start;
+ this.length = length;
+ }
+
+ private HudiBaseFile(HoodieBaseFile baseFile, long start, long length)
+ {
+ checkArgument(baseFile != null, "baseFile is null");
+ checkArgument(start >= 0, "start must be positive");
+ checkArgument(length >= 0, "length must be positive");
+ checkArgument(start + length <= baseFile.getFileSize(), "fileSize must be at least start + length");
+ this.path = baseFile.getPath();
+ this.fileName = baseFile.getFileName();
+ this.fileSize = baseFile.getFileSize();
+ this.modificationTime = baseFile.getPathInfo().getModificationTime();
+ this.start = start;
+ this.length = length;
+ }
+
+ @JsonProperty
+ @Override
+ public String getPath()
+ {
+ return path;
+ }
+
+ @JsonProperty
+ @Override
+ public String getFileName()
+ {
+ return fileName;
+ }
+
+ @JsonProperty
+ @Override
+ public long getFileSize()
+ {
+ return fileSize;
+ }
+
+ @JsonProperty
+ @Override
+ public long getModificationTime()
+ {
+ return modificationTime;
+ }
+
+ @JsonProperty
+ @Override
+ public long getStart()
+ {
+ return start;
+ }
+
+ @JsonProperty
+ @Override
+ public long getLength()
+ {
+ return length;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiFile.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiFile.java
new file mode 100644
index 000000000000..00e2ac66f9c7
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiFile.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.file;
+
+public interface HudiFile
+{
+ String getPath();
+
+ String getFileName();
+
+ long getFileSize();
+
+ long getModificationTime();
+
+ long getStart();
+
+ long getLength();
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java
new file mode 100644
index 000000000000..0ed071f23380
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/file/HudiLogFile.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.file;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hudi.common.model.HoodieLogFile;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+public class HudiLogFile
+ implements HudiFile
+{
+ private final String path;
+ private final String fileName;
+ private final long fileSize;
+ private final long modificationTime;
+ private final long start;
+ private final long length;
+
+ public static HudiLogFile of(HoodieLogFile logFile)
+ {
+ return of(logFile, 0, logFile.getFileSize());
+ }
+
+ public static HudiLogFile of(HoodieLogFile logFile, long start, long length)
+ {
+ return new HudiLogFile(logFile, start, length);
+ }
+
+ @JsonCreator
+ public HudiLogFile(@JsonProperty("path") String path,
+ @JsonProperty("fileName") String fileName,
+ @JsonProperty("fileSize") long fileSize,
+ @JsonProperty("modificationTime") long modificationTime,
+ @JsonProperty("start") long start,
+ @JsonProperty("length") long length)
+ {
+ this.path = path;
+ this.fileName = fileName;
+ this.fileSize = fileSize;
+ this.modificationTime = modificationTime;
+ this.start = start;
+ this.length = length;
+ }
+
+ private HudiLogFile(HoodieLogFile logFile, long start, long length)
+ {
+ checkArgument(logFile != null, "logFile is null");
+ checkArgument(start >= 0, "start must be positive");
+ checkArgument(length >= 0, "length must be positive");
+ checkArgument(start + length <= logFile.getFileSize(), "fileSize must be at least start + length");
+ this.path = logFile.getPath().toString();
+ this.fileName = logFile.getFileName();
+ this.fileSize = logFile.getFileSize();
+ this.modificationTime = logFile.getPathInfo().getModificationTime();
+ this.start = start;
+ this.length = length;
+ }
+
+ @JsonProperty
+ @Override
+ public String getPath()
+ {
+ return path;
+ }
+
+ @JsonProperty
+ @Override
+ public String getFileName()
+ {
+ return fileName;
+ }
+
+ @JsonProperty
+ @Override
+ public long getFileSize()
+ {
+ return fileSize;
+ }
+
+ @JsonProperty
+ @Override
+ public long getModificationTime()
+ {
+ return modificationTime;
+ }
+
+ @JsonProperty
+ @Override
+ public long getStart()
+ {
+ return start;
+ }
+
+ @JsonProperty
+ @Override
+ public long getLength()
+ {
+ return length;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java
new file mode 100644
index 000000000000..7e8bb967ffc7
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoFileReaderFactory.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.io;
+
+import org.apache.avro.Schema;
+import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.io.storage.HoodieAvroBootstrapFileReader;
+import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+
+import java.io.IOException;
+
+public class HudiTrinoFileReaderFactory
+ extends HoodieFileReaderFactory
+{
+ public HudiTrinoFileReaderFactory(HoodieStorage storage)
+ {
+ super(storage);
+ }
+
+ @Override
+ protected HoodieFileReader newParquetFileReader(StoragePath path)
+ {
+ throw new UnsupportedOperationException("HudiTrinoFileReaderFactory does not support Parquet file reader");
+ }
+
+ @Override
+ protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig,
+ StoragePath path,
+ Option<Schema> schemaOption)
+ throws IOException
+ {
+ return new HoodieNativeAvroHFileReader(storage, path, schemaOption);
+ }
+
+ @Override
+ protected HoodieFileReader newHFileFileReader(HoodieConfig hoodieConfig,
+ StoragePath path,
+ HoodieStorage storage,
+ byte[] content,
+ Option<Schema> schemaOption)
+ throws IOException
+ {
+ return new HoodieNativeAvroHFileReader(this.storage, content, schemaOption);
+ }
+
+ @Override
+ protected HoodieFileReader newOrcFileReader(StoragePath path)
+ {
+ throw new UnsupportedOperationException("HudiTrinoFileReaderFactory does not support ORC file reader");
+ }
+
+ @Override
+ public HoodieFileReader newBootstrapFileReader(HoodieFileReader skeletonFileReader, HoodieFileReader dataFileReader, Option<String[]> partitionFields, Object[] partitionValues)
+ {
+ return new HoodieAvroBootstrapFileReader(skeletonFileReader, dataFileReader, partitionFields, partitionValues);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java
new file mode 100644
index 000000000000..8044e92b58ec
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/HudiTrinoIOFactory.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.io;
+
+import org.apache.hudi.common.fs.ConsistencyGuard;
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.util.FileFormatUtils;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.io.storage.HoodieFileWriterFactory;
+import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+
+public class HudiTrinoIOFactory
+ extends HoodieIOFactory
+{
+ public HudiTrinoIOFactory(HoodieStorage storage)
+ {
+ super(storage);
+ }
+
+ @Override
+ public HoodieFileReaderFactory getReaderFactory(HoodieRecord.HoodieRecordType recordType)
+ {
+ return new HudiTrinoFileReaderFactory(storage);
+ }
+
+ @Override
+ public HoodieFileWriterFactory getWriterFactory(HoodieRecord.HoodieRecordType recordType)
+ {
+ throw new UnsupportedOperationException("HudiTrinoIOFactory does not support writers.");
+ }
+
+ @Override
+ public FileFormatUtils getFileFormatUtils(HoodieFileFormat fileFormat)
+ {
+ throw new UnsupportedOperationException("FileFormatUtils not supported in HudiTrinoIOFactory");
+ }
+
+ @Override
+ public HoodieStorage getStorage(StoragePath storagePath)
+ {
+ return storage;
+ }
+
+ @Override
+ public HoodieStorage getStorage(StoragePath path, boolean enableRetry, long maxRetryIntervalMs, int maxRetryNumbers, long initialRetryIntervalMs, String retryExceptions, ConsistencyGuard consistencyGuard)
+ {
+ return storage;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java
new file mode 100644
index 000000000000..5b1c322e0a39
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/InlineSeekableDataInputStream.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.io;
+
+import io.trino.filesystem.TrinoInputStream;
+
+import java.io.IOException;
+
+public class InlineSeekableDataInputStream
+ extends TrinoSeekableDataInputStream
+{
+ private final long startOffset;
+ private final long length;
+
+ public InlineSeekableDataInputStream(TrinoInputStream stream, long startOffset, long length)
+ {
+ super(stream);
+ this.startOffset = startOffset;
+ this.length = length;
+ }
+
+ @Override
+ public long getPos()
+ throws IOException
+ {
+ return super.getPos() - startOffset;
+ }
+
+ @Override
+ public void seek(long pos)
+ throws IOException
+ {
+ if (pos > length) {
+ throw new IOException(String.format(
+ "Attempting to seek past inline content: position to seek to is %s but the length is %s",
+ pos, length));
+ }
+ super.seek(startOffset + pos);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java
new file mode 100644
index 000000000000..babf44eba755
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/io/TrinoSeekableDataInputStream.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.io;
+
+import io.trino.filesystem.TrinoInputStream;
+import org.apache.hudi.io.SeekableDataInputStream;
+
+import java.io.IOException;
+
+public class TrinoSeekableDataInputStream
+ extends SeekableDataInputStream
+{
+ private final TrinoInputStream stream;
+
+ public TrinoSeekableDataInputStream(TrinoInputStream stream)
+ {
+ super(stream);
+ this.stream = stream;
+ }
+
+ @Override
+ public long getPos()
+ throws IOException
+ {
+ return stream.getPosition();
+ }
+
+ @Override
+ public void seek(long pos)
+ throws IOException
+ {
+ stream.seek(pos);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java
new file mode 100644
index 000000000000..f7e6fb1fa81b
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HiveHudiPartitionInfo.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.partition;
+
+import io.trino.filesystem.Location;
+import io.trino.metastore.Partition;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+
+import java.util.List;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static io.trino.plugin.hudi.HudiUtil.buildPartitionKeys;
+import static io.trino.plugin.hudi.HudiUtil.partitionMatchesPredicates;
+
+public class HiveHudiPartitionInfo
+ implements HudiPartitionInfo
+{
+ public static final String NON_PARTITION = "";
+
+ private final SchemaTableName schemaTableName;
+ private final List<HiveColumnHandle> partitionColumnHandles;
+ private final TupleDomain<HiveColumnHandle> constraintSummary;
+ private final String hivePartitionName;
+ private final String relativePartitionPath;
+ private final List<HivePartitionKey> hivePartitionKeys;
+
+ public HiveHudiPartitionInfo(
+ SchemaTableName schemaTableName,
+ Location tableLocation,
+ String hivePartitionName,
+ Partition partition,
+ List<HiveColumnHandle> partitionColumnHandles,
+ TupleDomain<HiveColumnHandle> constraintSummary)
+ {
+ this.schemaTableName = schemaTableName;
+ this.partitionColumnHandles = partitionColumnHandles;
+ this.constraintSummary = constraintSummary;
+ this.hivePartitionName = hivePartitionName;
+ this.relativePartitionPath = getRelativePartitionPath(
+ tableLocation,
+ Location.of(partition.getStorage().getLocation()));
+ this.hivePartitionKeys = buildPartitionKeys(
+ partitionColumnHandles, partition.getValues());
+ }
+
+ @Override
+ public String getRelativePartitionPath()
+ {
+ return relativePartitionPath;
+ }
+
+ @Override
+ public List<HivePartitionKey> getHivePartitionKeys()
+ {
+ return hivePartitionKeys;
+ }
+
+ @Override
+ public boolean doesMatchPredicates()
+ {
+ if (hivePartitionName.equals(NON_PARTITION)) {
+ return true;
+ }
+ return partitionMatchesPredicates(schemaTableName, hivePartitionName, partitionColumnHandles, constraintSummary);
+ }
+
+ public String getHivePartitionName()
+ {
+ return hivePartitionName;
+ }
+
+ private static String getRelativePartitionPath(Location baseLocation, Location fullPartitionLocation)
+ {
+ String basePath = baseLocation.path();
+ String fullPartitionPath = fullPartitionLocation.path();
+
+ if (!fullPartitionPath.startsWith(basePath)) {
+ throw new IllegalArgumentException("Partition location does not belong to base-location");
+ }
+
+ String baseLocationParent = baseLocation.parentDirectory().path();
+ String baseLocationName = baseLocation.fileName();
+ int partitionStartIndex = fullPartitionPath.indexOf(
+ baseLocationName,
+ baseLocationParent == null ? 0 : baseLocationParent.length());
+ // Partition-Path could be empty for non-partitioned tables
+ boolean isNonPartitionedTable = partitionStartIndex + baseLocationName.length() == fullPartitionPath.length();
+ return isNonPartitionedTable ? NON_PARTITION : fullPartitionPath.substring(partitionStartIndex + baseLocationName.length() + 1);
+ }
+
+ @Override
+ public String toString()
+ {
+ return toStringHelper(this)
+ .add("hivePartitionName", hivePartitionName)
+ .add("hivePartitionKeys", hivePartitionKeys)
+ .toString();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java
new file mode 100644
index 000000000000..1b607f1accf2
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfo.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.partition;
+
+import io.trino.plugin.hive.HivePartitionKey;
+
+import java.util.List;
+
+public interface HudiPartitionInfo
+{
+ String getRelativePartitionPath();
+
+ List<HivePartitionKey> getHivePartitionKeys();
+
+ boolean doesMatchPredicates();
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java
new file mode 100644
index 000000000000..874297b9b692
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/partition/HudiPartitionInfoLoader.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.partition;
+
+import io.airlift.concurrent.MoreFutures;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hive.util.AsyncQueue;
+import io.trino.plugin.hudi.query.HudiDirectoryLister;
+import io.trino.plugin.hudi.split.HudiSplitFactory;
+import io.trino.spi.connector.ConnectorSplit;
+import org.apache.hudi.common.model.FileSlice;
+
+import java.util.Deque;
+import java.util.List;
+import java.util.Optional;
+
+import static io.trino.plugin.hudi.partition.HiveHudiPartitionInfo.NON_PARTITION;
+
+public class HudiPartitionInfoLoader
+ implements Runnable
+{
+ private final HudiDirectoryLister hudiDirectoryLister;
+ private final HudiSplitFactory hudiSplitFactory;
+ private final AsyncQueue<ConnectorSplit> asyncQueue;
+ private final Deque<String> partitionQueue;
+ private final String commitTime;
+ private final boolean useIndex;
+
+ private boolean isRunning;
+
+ public HudiPartitionInfoLoader(
+ HudiDirectoryLister hudiDirectoryLister,
+ String commitTime,
+ HudiSplitFactory hudiSplitFactory,
+ AsyncQueue<ConnectorSplit> asyncQueue,
+ Deque<String> partitionQueue,
+ boolean useIndex)
+ {
+ this.hudiDirectoryLister = hudiDirectoryLister;
+ this.commitTime = commitTime;
+ this.hudiSplitFactory = hudiSplitFactory;
+ this.asyncQueue = asyncQueue;
+ this.partitionQueue = partitionQueue;
+ this.isRunning = true;
+ this.useIndex = useIndex;
+ }
+
+ @Override
+ public void run()
+ {
+ while (isRunning || !partitionQueue.isEmpty()) {
+ String partitionName = partitionQueue.poll();
+
+ if (partitionName != null) {
+ generateSplitsFromPartition(partitionName);
+ }
+ }
+ }
+
+ private void generateSplitsFromPartition(String partitionName)
+ {
+ Optional<HudiPartitionInfo> partitionInfo = hudiDirectoryLister.getPartitionInfo(partitionName);
+ partitionInfo.ifPresent(hudiPartitionInfo -> {
+ if (hudiPartitionInfo.doesMatchPredicates() || partitionName.equals(NON_PARTITION)) {
+ List<HivePartitionKey> partitionKeys = hudiPartitionInfo.getHivePartitionKeys();
+ List<FileSlice> partitionFileSlices = hudiDirectoryLister.listStatus(hudiPartitionInfo, useIndex);
+ partitionFileSlices.stream()
+ .flatMap(slice -> hudiSplitFactory.createSplits(partitionKeys, slice, commitTime).stream())
+ .map(asyncQueue::offer)
+ .forEachOrdered(MoreFutures::getFutureValue);
+ }
+ });
+ }
+
+ public void stopRunning()
+ {
+ this.isRunning = false;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java
new file mode 100644
index 000000000000..fb8f1903358e
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiDirectoryLister.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query;
+
+import io.trino.plugin.hudi.partition.HudiPartitionInfo;
+import org.apache.hudi.common.model.FileSlice;
+
+import java.io.Closeable;
+import java.util.List;
+import java.util.Optional;
+
+public interface HudiDirectoryLister
+ extends Closeable
+{
+ List<FileSlice> listStatus(HudiPartitionInfo partitionInfo, boolean useIndex);
+
+ Optional<HudiPartitionInfo> getPartitionInfo(String partition);
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java
new file mode 100644
index 000000000000..dfd7ef7d3c39
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/HudiSnapshotDirectoryLister.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query;
+
+import com.google.common.collect.ImmutableList;
+import io.airlift.log.Logger;
+import io.trino.filesystem.Location;
+import io.trino.metastore.Partition;
+import io.trino.plugin.hudi.HudiTableHandle;
+import io.trino.plugin.hudi.partition.HiveHudiPartitionInfo;
+import io.trino.plugin.hudi.partition.HudiPartitionInfo;
+import io.trino.plugin.hudi.query.index.HudiIndexSupport;
+import io.trino.plugin.hudi.query.index.IndexSupportFactory;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.trino.plugin.hudi.HudiUtil.getFileSystemView;
+
+public class HudiSnapshotDirectoryLister
+ implements HudiDirectoryLister
+{
+ private static final Logger log = Logger.get(HudiSnapshotDirectoryLister.class);
+ private final HudiTableHandle tableHandle;
+ private final Lazy<HoodieTableFileSystemView> lazyFileSystemView;
+ private final Lazy<Map<String, HudiPartitionInfo>> lazyAllPartitionInfoMap;
+ private final Optional<HudiIndexSupport> indexSupportOpt;
+
+ public HudiSnapshotDirectoryLister(
+ ConnectorSession session,
+ HudiTableHandle tableHandle,
+ boolean enableMetadataTable,
+ Lazy<HoodieTableMetadata> lazyTableMetadata,
+ Lazy<Map<String, Partition>> lazyAllPartitions)
+ {
+ this.tableHandle = tableHandle;
+ SchemaTableName schemaTableName = tableHandle.getSchemaTableName();
+ this.lazyFileSystemView = Lazy.lazily(() -> {
+ HoodieTimer timer = HoodieTimer.start();
+ HoodieTableMetaClient metaClient = tableHandle.getMetaClient();
+ HoodieTableFileSystemView fileSystemView = getFileSystemView(lazyTableMetadata.get(), metaClient);
+ if (enableMetadataTable) {
+ fileSystemView.loadAllPartitions();
+ }
+ log.info("Created file system view of table %s in %s ms", schemaTableName, timer.endTimer());
+ return fileSystemView;
+ });
+
+ this.lazyAllPartitionInfoMap = Lazy.lazily(() -> lazyAllPartitions.get().entrySet().stream()
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ e -> new HiveHudiPartitionInfo(
+ schemaTableName,
+ Location.of(tableHandle.getBasePath()),
+ e.getKey(),
+ e.getValue(),
+ tableHandle.getPartitionColumns(),
+ tableHandle.getPartitionPredicates()))));
+ Lazy<HoodieTableMetaClient> lazyMetaClient = Lazy.lazily(tableHandle::getMetaClient);
+ this.indexSupportOpt = enableMetadataTable ?
+ IndexSupportFactory.createIndexSupport(schemaTableName, lazyMetaClient, lazyTableMetadata, tableHandle.getRegularPredicates(), session) : Optional.empty();
+ }
+
+ @Override
+ public List<FileSlice> listStatus(HudiPartitionInfo partitionInfo, boolean useIndex)
+ {
+ HoodieTimer timer = HoodieTimer.start();
+ Stream<FileSlice> slices = lazyFileSystemView.get().getLatestFileSlicesBeforeOrOn(
+ partitionInfo.getRelativePartitionPath(),
+ tableHandle.getLatestCommitTime(),
+ false);
+
+ if (!useIndex) {
+ return slices.collect(toImmutableList());
+ }
+
+ ImmutableList<FileSlice> collect = slices
+ .filter(slice -> indexSupportOpt
+ .map(indexSupport -> !indexSupport.shouldSkipFileSlice(slice))
+ .orElse(true))
+ .collect(toImmutableList());
+ log.info("Listed partition [%s] on table %s.%s in %s ms",
+ partitionInfo, tableHandle.getSchemaName(), tableHandle.getTableName(), timer.endTimer());
+ return collect;
+ }
+
+ @Override
+ public Optional<HudiPartitionInfo> getPartitionInfo(String partition)
+ {
+ return Optional.ofNullable(lazyAllPartitionInfoMap.get().get(partition));
+ }
+
+ @Override
+ public void close()
+ {
+ if (!lazyFileSystemView.get().isClosed()) {
+ lazyFileSystemView.get().close();
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java
new file mode 100644
index 000000000000..044908f9c49f
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiBaseIndexSupport.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.trino.spi.connector.SchemaTableName;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieIndexDefinition;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
+
+public abstract class HudiBaseIndexSupport
+ implements HudiIndexSupport
+{
+ private final Logger log;
+ protected final SchemaTableName schemaTableName;
+ protected final Lazy<HoodieTableMetaClient> lazyMetaClient;
+
+ public HudiBaseIndexSupport(Logger log, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient)
+ {
+ this.log = requireNonNull(log, "log is null");
+ this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null");
+ this.lazyMetaClient = requireNonNull(lazyMetaClient, "metaClient is null");
+ }
+
+ public void printDebugMessage(Map<String, List<FileSlice>> candidateFileSlices, Map<String, List<FileSlice>> inputFileSlices, long lookupDurationMs)
+ {
+ if (log.isDebugEnabled()) {
+ int candidateFileSize = candidateFileSlices.values().stream().mapToInt(List::size).sum();
+ int totalFiles = inputFileSlices.values().stream().mapToInt(List::size).sum();
+ double skippingPercent = totalFiles == 0 ? 0.0d : (totalFiles - candidateFileSize) / (totalFiles * 1.0d);
+
+ log.info("Total files: %s; files after data skipping: %s; skipping percent %s; time taken: %s ms; table name: %s",
+ totalFiles,
+ candidateFileSize,
+ skippingPercent,
+ lookupDurationMs,
+ schemaTableName);
+ }
+ }
+
+ protected Map<String, HoodieIndexDefinition> getAllIndexDefinitions()
+ {
+ if (lazyMetaClient.get().getIndexMetadata().isEmpty()) {
+ return Map.of();
+ }
+
+ return lazyMetaClient.get().getIndexMetadata().get().getIndexDefinitions();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
new file mode 100644
index 000000000000..72e46693ee92
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import com.google.common.collect.ImmutableList;
+import io.airlift.log.Logger;
+import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
+import io.airlift.units.Duration;
+import io.trino.parquet.predicate.TupleDomainParquetPredicate;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.TupleDomainUtils;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.Domain;
+import io.trino.spi.predicate.Range;
+import io.trino.spi.predicate.TupleDomain;
+import io.trino.spi.predicate.ValueSet;
+import io.trino.spi.type.Type;
+import io.trino.spi.type.VarcharType;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
+import org.apache.hudi.common.model.BaseFile;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieIndexDefinition;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.hash.ColumnIndexID;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static io.trino.parquet.predicate.PredicateUtils.isStatisticsOverflow;
+import static io.trino.plugin.hudi.HudiSessionProperties.getColumnStatsWaitTimeout;
+import static io.trino.spi.type.BigintType.BIGINT;
+import static io.trino.spi.type.BooleanType.BOOLEAN;
+import static io.trino.spi.type.DateType.DATE;
+import static io.trino.spi.type.DoubleType.DOUBLE;
+import static io.trino.spi.type.IntegerType.INTEGER;
+import static io.trino.spi.type.RealType.REAL;
+import static io.trino.spi.type.SmallintType.SMALLINT;
+import static io.trino.spi.type.TinyintType.TINYINT;
+import static java.lang.Float.floatToRawIntBits;
+
+public class HudiColumnStatsIndexSupport
+ extends HudiBaseIndexSupport
+{
+ private static final Logger log = Logger.get(HudiColumnStatsIndexSupport.class);
+ private final CompletableFuture<Optional<Map<String, Map<String, HoodieMetadataColumnStats>>>> statsByFileNameFuture;
+ protected final TupleDomain<String> regularColumnPredicates;
+ private final List<String> regularColumns;
+ private final Duration columnStatsWaitTimeout;
+ private final long futureStartTimeMs;
+
+ public HudiColumnStatsIndexSupport(ConnectorSession session, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient, Lazy<HoodieTableMetadata> lazyTableMetadata, TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ this(log, session, schemaTableName, lazyMetaClient, lazyTableMetadata, regularColumnPredicates);
+ }
+
+ public HudiColumnStatsIndexSupport(Logger log, ConnectorSession session, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient, Lazy<HoodieTableMetadata> lazyTableMetadata, TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ super(log, schemaTableName, lazyMetaClient);
+ this.columnStatsWaitTimeout = getColumnStatsWaitTimeout(session);
+ this.regularColumnPredicates = regularColumnPredicates.transformKeys(HiveColumnHandle::getName);
+ this.regularColumns = this.regularColumnPredicates
+ .getDomains().get().entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
+ if (regularColumnPredicates.isAll() || !regularColumnPredicates.getDomains().isPresent()) {
+ this.statsByFileNameFuture = CompletableFuture.completedFuture(Optional.empty());
+ }
+ else {
+ // Get filter columns
+ List<String> encodedTargetColumnNames = regularColumns
+ .stream()
+ .map(col -> new ColumnIndexID(col).asBase64EncodedString()).collect(Collectors.toList());
+ statsByFileNameFuture = CompletableFuture.supplyAsync(() -> {
+ HoodieTimer timer = HoodieTimer.start();
+ if (!lazyMetaClient.get().getTableConfig().getMetadataPartitions()
+ .contains(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS)) {
+ return Optional.empty();
+ }
+
+ Map<String, Map<String, HoodieMetadataColumnStats>> statsByFileName =
+ lazyTableMetadata.get().getRecordsByKeyPrefixes(encodedTargetColumnNames,
+ HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS, true)
+ .collectAsList()
+ .stream()
+ .filter(f -> f.getData().getColumnStatMetadata().isPresent())
+ .map(f -> f.getData().getColumnStatMetadata().get())
+ .collect(Collectors.groupingBy(
+ HoodieMetadataColumnStats::getFileName,
+ Collectors.toMap(
+ HoodieMetadataColumnStats::getColumnName,
+ Function.identity())));
+ log.debug("Column stats lookup took %s ms and identified %d relevant file IDs.", timer.endTimer(), statsByFileName.size());
+
+ return Optional.of(statsByFileName);
+ });
+ }
+ this.futureStartTimeMs = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean shouldSkipFileSlice(FileSlice slice)
+ {
+ try {
+ if (statsByFileNameFuture.isDone()) {
+ Optional<Map<String, Map<String, HoodieMetadataColumnStats>>> statsOpt = statsByFileNameFuture.get();
+ return statsOpt
+ .map(stats -> shouldSkipFileSlice(slice, stats, regularColumnPredicates, regularColumns))
+ .orElse(false);
+ }
+
+ long elapsedMs = System.currentTimeMillis() - futureStartTimeMs;
+ if (elapsedMs > columnStatsWaitTimeout.toMillis()) {
+ // Took too long; skip decision
+ return false;
+ }
+
+ // If still within the timeout window, wait up to the remaining time
+ long remainingMs = Math.max(0, columnStatsWaitTimeout.toMillis() - elapsedMs);
+ Optional<Map<String, Map<String, HoodieMetadataColumnStats>>> statsOpt =
+ statsByFileNameFuture.get(remainingMs, TimeUnit.MILLISECONDS);
+
+ return statsOpt
+ .map(stats -> shouldSkipFileSlice(slice, stats, regularColumnPredicates, regularColumns))
+ .orElse(false);
+ }
+ catch (TimeoutException | InterruptedException | ExecutionException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public boolean canApply(TupleDomain<String> tupleDomain)
+ {
+ boolean isIndexSupported = isIndexSupportAvailable();
+ // indexDefinition is only available after table version EIGHT
+ // For tables that have versions < EIGHT, column stats index is available as long as partition in metadata is available
+ if (!isIndexSupported || lazyMetaClient.get().getTableConfig().getTableVersion().lesserThan(HoodieTableVersion.EIGHT)) {
+ log.debug("Column Stats Index partition is not enabled in metadata.");
+ return isIndexSupported;
+ }
+
+ Map<String, HoodieIndexDefinition> indexDefinitions = getAllIndexDefinitions();
+ HoodieIndexDefinition colStatsDefinition = indexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS);
+ if (colStatsDefinition == null || colStatsDefinition.getSourceFields() == null || colStatsDefinition.getSourceFields().isEmpty()) {
+ log.warn("Column stats index definition is missing or has no source fields defined");
+ return false;
+ }
+
+ // Optimization applied: Only consider applicable if predicates reference indexed columns
+ List<String> sourceFields = colStatsDefinition.getSourceFields();
+ boolean applicable = TupleDomainUtils.areSomeFieldsReferenced(tupleDomain, sourceFields);
+
+ if (applicable) {
+ log.debug("Column Stats Index is available and applicable (predicates reference indexed columns).");
+ }
+ else {
+ log.debug("Column Stats Index is available, but predicates do not reference any indexed columns.");
+ }
+ return applicable;
+ }
+
+ public boolean isIndexSupportAvailable()
+ {
+ return lazyMetaClient.get().getTableConfig().getMetadataPartitions()
+ .contains(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS);
+ }
+
+ // TODO: Move helper functions below to TupleDomain/DomainUtils
+ private static boolean shouldSkipFileSlice(
+ FileSlice fileSlice,
+ Map<String, Map<String, HoodieMetadataColumnStats>> statsByFileName,
+ TupleDomain<String> regularColumnPredicates,
+ List<String> regularColumns)
+ {
+ String fileSliceName = fileSlice.getBaseFile().map(BaseFile::getFileName).orElse("");
+ // If no stats exist for this specific file, we cannot prune it.
+ if (!statsByFileName.containsKey(fileSliceName)) {
+ return false;
+ }
+ Map<String, HoodieMetadataColumnStats> stats = statsByFileName.get(fileSliceName);
+ return !evaluateStatisticPredicate(regularColumnPredicates, stats, regularColumns);
+ }
+
+ protected static boolean evaluateStatisticPredicate(
+ TupleDomain<String> regularColumnPredicates,
+ Map<String, HoodieMetadataColumnStats> stats,
+ List<String> regularColumns)
+ {
+ if (regularColumnPredicates.isNone() || !regularColumnPredicates.getDomains().isPresent()) {
+ return true;
+ }
+ for (String regularColumn : regularColumns) {
+ Domain columnPredicate = regularColumnPredicates.getDomains().get().get(regularColumn);
+ Optional<HoodieMetadataColumnStats> currentColumnStats = Optional.ofNullable(stats.get(regularColumn));
+ if (currentColumnStats.isEmpty()) {
+ // No stats for column
+ }
+ else {
+ Domain domain = getDomain(regularColumn, columnPredicate.getType(), currentColumnStats.get());
+ if (columnPredicate.intersect(domain).isNone()) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static Domain getDomain(String colName, Type type, HoodieMetadataColumnStats statistics)
+ {
+ if (statistics == null) {
+ return Domain.all(type);
+ }
+ boolean hasNullValue = statistics.getNullCount() != 0L;
+ boolean hasNonNullValue = statistics.getValueCount() - statistics.getNullCount() > 0;
+ if (!hasNonNullValue || statistics.getMaxValue() == null || statistics.getMinValue() == null) {
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ if (!(statistics.getMinValue() instanceof GenericRecord) ||
+ !(statistics.getMaxValue() instanceof GenericRecord)) {
+ return Domain.all(type);
+ }
+ return getDomain(colName, type, ((GenericRecord) statistics.getMinValue()).get(0),
+ ((GenericRecord) statistics.getMaxValue()).get(0), hasNullValue);
+ }
+
+ /**
+ * Get a domain for the ranges defined by each pair of elements from {@code minimums} and {@code maximums}.
+ * Both arrays must have the same length.
+ */
+ private static Domain getDomain(String colName, Type type, Object minimum, Object maximum, boolean hasNullValue)
+ {
+ try {
+ if (type.equals(BOOLEAN)) {
+ boolean hasTrueValue = (boolean) minimum || (boolean) maximum;
+ boolean hasFalseValue = !(boolean) minimum || !(boolean) maximum;
+ if (hasTrueValue && hasFalseValue) {
+ return Domain.all(type);
+ }
+ if (hasTrueValue) {
+ return Domain.create(ValueSet.of(type, true), hasNullValue);
+ }
+ if (hasFalseValue) {
+ return Domain.create(ValueSet.of(type, false), hasNullValue);
+ }
+ // No other case, since all null case is handled earlier.
+ }
+
+ if ((type.equals(BIGINT) || type.equals(TINYINT) || type.equals(SMALLINT)
+ || type.equals(INTEGER) || type.equals(DATE))) {
+ long minValue = TupleDomainParquetPredicate.asLong(minimum);
+ long maxValue = TupleDomainParquetPredicate.asLong(maximum);
+ if (isStatisticsOverflow(type, minValue, maxValue)) {
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ return ofMinMax(type, minValue, maxValue, hasNullValue);
+ }
+
+ if (type.equals(REAL)) {
+ Float minValue = (Float) minimum;
+ Float maxValue = (Float) maximum;
+ if (minValue.isNaN() || maxValue.isNaN()) {
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ return ofMinMax(type, (long) floatToRawIntBits(minValue), (long) floatToRawIntBits(maxValue), hasNullValue);
+ }
+
+ if (type.equals(DOUBLE)) {
+ Double minValue = (Double) minimum;
+ Double maxValue = (Double) maximum;
+ if (minValue.isNaN() || maxValue.isNaN()) {
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ return ofMinMax(type, minValue, maxValue, hasNullValue);
+ }
+
+ if (type.equals(VarcharType.VARCHAR)) {
+ Slice min = Slices.utf8Slice((String) minimum);
+ Slice max = Slices.utf8Slice((String) maximum);
+ return ofMinMax(type, min, max, hasNullValue);
+ }
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ catch (Exception e) {
+ log.warn("failed to create Domain for column: %s which type is: %s", colName, type.toString());
+ return Domain.create(ValueSet.all(type), hasNullValue);
+ }
+ }
+
+ private static Domain ofMinMax(Type type, Object min, Object max, boolean hasNullValue)
+ {
+ Range range = Range.range(type, min, true, max, true);
+ ValueSet vs = ValueSet.ofRanges(ImmutableList.of(range));
+ return Domain.create(vs, hasNullValue);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java
new file mode 100644
index 000000000000..de33640f4890
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiIndexSupport.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.model.FileSlice;
+
+public interface HudiIndexSupport
+{
+ boolean canApply(TupleDomain<String> tupleDomain);
+
+ default boolean shouldSkipFileSlice(FileSlice slice)
+ {
+ return false;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java
new file mode 100644
index 000000000000..e79eb342812b
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiNoOpIndexSupport.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.util.Lazy;
+
+/**
+ * Noop index support to ensure that MDT enabled split generation is entered.
+ */
+public class HudiNoOpIndexSupport
+ extends HudiBaseIndexSupport
+{
+ private static final Logger log = Logger.get(HudiNoOpIndexSupport.class);
+
+ public HudiNoOpIndexSupport(SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient)
+ {
+ super(log, schemaTableName, lazyMetaClient);
+ }
+
+ @Override
+ public boolean canApply(TupleDomain<String> tupleDomain)
+ {
+ return true;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java
new file mode 100644
index 000000000000..c2e98820fb6f
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiPartitionStatsIndexSupport.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.TupleDomainUtils;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
+import org.apache.hudi.common.model.HoodieIndexDefinition;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.hash.ColumnIndexID;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.util.Lazy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static io.trino.plugin.hudi.util.TupleDomainUtils.hasSimpleNullCheck;
+
+public class HudiPartitionStatsIndexSupport
+ extends HudiColumnStatsIndexSupport
+{
+ private static final Logger log = Logger.get(HudiColumnStatsIndexSupport.class);
+ private final Lazy<HoodieTableMetadata> lazyMetadataTable;
+
+ public HudiPartitionStatsIndexSupport(ConnectorSession session, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient, Lazy<HoodieTableMetadata> lazyTableMetadata, TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ super(log, session, schemaTableName, lazyMetaClient, lazyTableMetadata, regularColumnPredicates);
+ this.lazyMetadataTable = lazyTableMetadata;
+ }
+
+ public Optional<List<String>> prunePartitions(
+ List<String> allPartitions)
+ {
+ HoodieTimer timer = HoodieTimer.start();
+
+ // Filter out predicates containing simple null checks (`IS NULL` or `IS NOT NULL`)
+ TupleDomain<String> filteredRegularPredicates = regularColumnPredicates.filter((_, domain) -> !hasSimpleNullCheck(domain));
+
+ // Sanity check, if no regular domains, return immediately
+ if (filteredRegularPredicates.getDomains().isEmpty()) {
+ timer.endTimer();
+ return Optional.empty();
+ }
+
+ List<String> regularColumns = new ArrayList<>(filteredRegularPredicates.getDomains().get().keySet());
+
+ // Get columns to filter on
+ List<String> encodedTargetColumnNames = regularColumns
+ .stream()
+ .map(col -> new ColumnIndexID(col).asBase64EncodedString()).toList();
+
+ // Map of partition stats keyed by partition name
+ Map<String, Map<String, HoodieMetadataColumnStats>> statsByPartitionName = lazyMetadataTable.get().getRecordsByKeyPrefixes(
+ encodedTargetColumnNames,
+ HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS, true)
+ .collectAsList()
+ .stream()
+ .filter(f -> f.getData().getColumnStatMetadata().isPresent())
+ .map(f -> f.getData().getColumnStatMetadata().get())
+ .collect(Collectors.groupingBy(
+ HoodieMetadataColumnStats::getFileName,
+ Collectors.toMap(
+ HoodieMetadataColumnStats::getColumnName,
+ Function.identity())));
+
+ // For each partition, determine if it should be kept based on stats availability and predicate evaluation
+ List<String> prunedPartitions = allPartitions.stream()
+ .filter(partition -> {
+ // Check if stats exist for this partition
+ Map<String, HoodieMetadataColumnStats> partitionStats = statsByPartitionName.get(partition);
+ if (partitionStats == null) {
+ // Partition has no stats in the index, keep it
+ return true;
+ }
+ else {
+ // Partition has stats, evaluate the predicate against them
+ // Keep the partition only if the predicate evaluates to true
+ // Important: If some columns in encodedTargetColumnNames is not available in partition stats, partition will not be pruned iff all available predicate
+ // evaluates to true. Since we cannot determine if the predicate will evaluate to true or not on the missing stat, adopt conservative measure to true,
+ // i.e. to not prune
+ return evaluateStatisticPredicate(filteredRegularPredicates, partitionStats, regularColumns);
+ }
+ })
+ .collect(Collectors.toList());
+
+ log.info("Took %s ms to prune partitions using Partition Stats Index for table %s", timer.endTimer(), schemaTableName);
+ return Optional.of(prunedPartitions);
+ }
+
+ @Override
+ public boolean isIndexSupportAvailable()
+ {
+ return lazyMetaClient.get().getTableConfig().getMetadataPartitions()
+ .contains(HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS);
+ }
+
+ @Override
+ public boolean canApply(TupleDomain<String> tupleDomain)
+ {
+ // Important: has the same implementation as col stats superclass, only difference is that log messages are different
+ if (!isIndexSupportAvailable()) {
+ log.debug("Partition Stats Index partition is not enabled in metadata table.");
+ return false;
+ }
+
+ Map<String, HoodieIndexDefinition> indexDefinitions = getAllIndexDefinitions();
+ HoodieIndexDefinition partitionsStatsIndex = indexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS);
+ if (partitionsStatsIndex == null || partitionsStatsIndex.getSourceFields() == null || partitionsStatsIndex.getSourceFields().isEmpty()) {
+ log.warn("Partition stats index definition is missing or has no source fields defined");
+ return false;
+ }
+
+ // Optimization applied: Only consider applicable if predicates reference indexed columns
+ List<String> sourceFields = partitionsStatsIndex.getSourceFields();
+ boolean applicable = TupleDomainUtils.areSomeFieldsReferenced(tupleDomain, sourceFields);
+
+ if (applicable) {
+ log.debug("Partition Stats Index is available and applicable (predicates reference indexed columns).");
+ }
+ else {
+ log.debug("Partition Stats Index is available, but predicates do not reference any indexed columns.");
+ }
+ return applicable;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java
new file mode 100644
index 000000000000..3c5e33f4e6a0
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiRecordLevelIndexSupport.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.airlift.slice.Slice;
+import io.airlift.units.Duration;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.TupleDomainUtils;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.Domain;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.util.Lazy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_BAD_DATA;
+import static io.trino.plugin.hudi.HudiSessionProperties.getRecordIndexWaitTimeout;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public class HudiRecordLevelIndexSupport
+ extends HudiBaseIndexSupport
+{
+ private static final Logger log = Logger.get(HudiRecordLevelIndexSupport.class);
+
+ public static final String DEFAULT_COLUMN_VALUE_SEPARATOR = ":";
+ public static final String DEFAULT_RECORD_KEY_PARTS_SEPARATOR = ",";
+ private final CompletableFuture<Optional<Set<String>>> relevantFileIdsFuture;
+ private final Duration recordIndexWaitTimeout;
+ private final long futureStartTimeMs;
+
+ public HudiRecordLevelIndexSupport(ConnectorSession session, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient, Lazy<HoodieTableMetadata> lazyTableMetadata, TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ super(log, schemaTableName, lazyMetaClient);
+ this.recordIndexWaitTimeout = getRecordIndexWaitTimeout(session);
+ if (regularColumnPredicates.isAll()) {
+ log.debug("Predicates cover all data, skipping record level index lookup.");
+ this.relevantFileIdsFuture = CompletableFuture.completedFuture(Optional.empty());
+ }
+ else {
+ this.relevantFileIdsFuture = CompletableFuture.supplyAsync(() -> {
+ HoodieTimer timer = HoodieTimer.start();
+ Option<String[]> recordKeyFieldsOpt = lazyMetaClient.get().getTableConfig().getRecordKeyFields();
+ if (recordKeyFieldsOpt.isEmpty() || recordKeyFieldsOpt.get().length == 0) {
+ // Should not happen since canApply checks for this, include for safety
+ throw new TrinoException(HUDI_BAD_DATA, "Record key fields must be defined to use Record Level Index.");
+ }
+ List<String> recordKeyFields = Arrays.asList(recordKeyFieldsOpt.get());
+
+ TupleDomain<String> regularPredicatesTransformed = regularColumnPredicates.transformKeys(HiveColumnHandle::getName);
+ // Only extract the predicates relevant to the record key fields
+ TupleDomain<String> filteredDomains = extractPredicatesForColumns(regularPredicatesTransformed, recordKeyFields);
+
+ // Construct the actual record keys based on the filtered predicates using Hudi's encoding scheme
+ List<String> recordKeys = constructRecordKeys(filteredDomains, recordKeyFields);
+
+ if (recordKeys.isEmpty()) {
+ // If key construction fails (e.g., incompatible predicates not caught by canApply, or placeholder issue)
+ log.warn("Took %s ms, but could not construct record keys from predicates. Skipping record index pruning for table %s",
+ timer.endTimer(), schemaTableName);
+ return Optional.empty();
+ }
+ log.debug(String.format("Constructed %d record keys for index lookup.", recordKeys.size()));
+
+ // Perform index lookup in metadataTable
+ // TODO: document here what this map is keyed by
+ Map<String, HoodieRecordGlobalLocation> recordIndex = lazyTableMetadata.get().readRecordIndex(recordKeys);
+ if (recordIndex.isEmpty()) {
+ log.debug("Record level index lookup took %s ms but returned no locations for the given keys %s for table %s",
+ timer.endTimer(), recordKeys, schemaTableName);
+ // Return all original fileSlices
+ return Optional.empty();
+ }
+
+ // Collect fileIds for pruning
+ Set<String> relevantFiles = recordIndex.values().stream()
+ .map(HoodieRecordGlobalLocation::getFileId)
+ .collect(Collectors.toSet());
+ log.debug("Record level index lookup took %s ms and identified %d relevant file IDs.", timer.endTimer(), relevantFiles.size());
+
+ return Optional.of(relevantFiles);
+ });
+ }
+ this.futureStartTimeMs = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean shouldSkipFileSlice(FileSlice slice)
+ {
+ try {
+ if (relevantFileIdsFuture.isDone()) {
+ Optional<Set<String>> relevantFileIds = relevantFileIdsFuture.get();
+ return relevantFileIds.map(fileIds -> !fileIds.contains(slice.getFileId())).orElse(false);
+ }
+
+ long elapsedMs = System.currentTimeMillis() - futureStartTimeMs;
+ if (elapsedMs > recordIndexWaitTimeout.toMillis()) {
+ // Took too long; skip decision
+ return false;
+ }
+
+ long remainingMs = Math.max(0, recordIndexWaitTimeout.toMillis() - elapsedMs);
+ Optional<Set<String>> relevantFileIds = relevantFileIdsFuture.get(remainingMs, MILLISECONDS);
+ return relevantFileIds.map(fileIds -> !fileIds.contains(slice.getFileId())).orElse(false);
+ }
+ catch (TimeoutException | InterruptedException | ExecutionException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Checks if the Record Level Index is available and the query predicates
+ * reference all record key fields with compatible (IN/EQUAL) constraints.
+ */
+ @Override
+ public boolean canApply(TupleDomain<String> tupleDomain)
+ {
+ if (!isIndexSupportAvailable()) {
+ log.debug("Record Level Index partition is not enabled in metadata.");
+ return false;
+ }
+
+ Option<String[]> recordKeyFieldsOpt = lazyMetaClient.get().getTableConfig().getRecordKeyFields();
+ if (recordKeyFieldsOpt.isEmpty() || recordKeyFieldsOpt.get().length == 0) {
+ log.debug("Record key fields are not defined in table config.");
+ return false;
+ }
+ List<String> recordKeyFields = Arrays.asList(recordKeyFieldsOpt.get());
+
+ // Ensure that predicates reference all record key fields and use IN/EQUAL
+ boolean applicable = TupleDomainUtils.areAllFieldsReferenced(tupleDomain, recordKeyFields)
+ && TupleDomainUtils.areDomainsInOrEqualOnly(tupleDomain, recordKeyFields);
+
+ if (!applicable) {
+ log.debug("Predicates do not reference all record key fields or use non-compatible (non IN/EQUAL) constraints.");
+ }
+ else {
+ log.debug("Record Level Index is available and applicable based on predicates.");
+ }
+ return applicable;
+ }
+
+ private boolean isIndexSupportAvailable()
+ {
+ return lazyMetaClient.get().getTableConfig().getMetadataPartitions()
+ .contains(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX);
+ }
+
+ /**
+ * Extracts predicates from a TupleDomain that match a given set of columns.
+ * Preserves all complex predicate properties including multi-value domains,
+ * range-based predicates, and nullability.
+ *
+ * @param tupleDomain The source TupleDomain containing all predicates
+ * @param columnFields The set of columns for which to extract predicates
+ * @return A new TupleDomain containing only the predicates for the specified columns
+ */
+ public static TupleDomain<String> extractPredicatesForColumns(TupleDomain<String> tupleDomain, List<String> columnFields)
+ {
+ if (tupleDomain.isNone()) {
+ return TupleDomain.none();
+ }
+
+ if (tupleDomain.isAll()) {
+ return TupleDomain.all();
+ }
+
+ // Extract the domains matching the specified columns
+ Map<String, Domain> allDomains = tupleDomain.getDomains().get();
+ Map<String, Domain> filteredDomains = allDomains.entrySet().stream().filter(entry -> columnFields.contains(entry.getKey())) // Ensure key is in the column set
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ // If no domains matched, but we had some columns to extract, return ALL
+ if (filteredDomains.isEmpty() && !columnFields.isEmpty() && !allDomains.isEmpty()) {
+ return TupleDomain.all();
+ }
+
+ return TupleDomain.withColumnDomains(filteredDomains);
+ }
+
+ /**
+ * Constructs a record key from TupleDomain based on whether it's a complex key or not.
+ * <p>
+ * Construction of record keys will only be handled for domains generated from EQUALITY or IN predicates.
+ * <p>
+ * An empty list of record keys will be generated if the following conditions are not met:
+ * <ol>
+ * <li>recordKeysFields is empty</li>
+ * <li>recordKeyDomains isAll</li>
+ * <li>For the case of complex key, domains are not applied to all recordKeysFields</li>
+ * <li>For the case of complex key, domains are applied to all recordKeyFields, but one of the domain is <b>NOT</b>
+ * generated from an equality or IN predicate</li>
+ * </ol>
+ * <p>
+ * Note: This function is O(m^n) where m is the average size of value literals and n is the number of record keys.
+ * <p>
+ * Optimization 1: If MDT enabled functions allows for streams to be passed in, we can implement an iterator to be more memory efficient.
+ * <p>
+ * Optimization 2: We should also consider limiting the number of recordKeys generated, if it is estimated to be more than a limit, RLI should just be skipped
+ * as it may just be faster to read out all data and filer accordingly.
+ *
+ * @param recordKeyDomains The filtered TupleDomain containing column handles and values
+ * @param recordKeyFields List of column names that represent the record keys
+ * @return List of string values representing the record key(s)
+ */
+ public static List<String> constructRecordKeys(TupleDomain<String> recordKeyDomains, List<String> recordKeyFields)
+ {
+ // TODO: Move this to TupleDomainUtils
+ // If no recordKeys or no recordKeyDomains, return empty list
+ if (recordKeyFields == null || recordKeyFields.isEmpty() || recordKeyDomains.isAll()) {
+ return Collections.emptyList();
+ }
+
+ // All recordKeys must have a domain else, return empty list (applicable to complexKeys)
+ // If a one of the recordKey in the set of complexKeys does not have a domain, we are unable to construct
+ // a complete complexKey
+ if (!recordKeyDomains.getDomains().get().keySet().containsAll(recordKeyFields)) {
+ return Collections.emptyList();
+ }
+
+ // Extract the domain mappings from the tuple domain
+ Map<String, Domain> domains = recordKeyDomains.getDomains().get();
+
+ // Case 1: Not a complex key (single record key)
+ if (recordKeyFields.size() == 1) {
+ String recordKey = recordKeyFields.getFirst();
+
+ // Extract value for this key
+ Domain domain = domains.get(recordKey);
+ return extractStringValues(domain);
+ }
+ // Case 2: Complex/Composite key (multiple record keys)
+ else {
+ // Create a queue to manage the Cartesian product generation
+ Queue<String> results = new LinkedList<>();
+
+ // For each key in the complex key
+ for (String recordKeyField : recordKeyFields) {
+ // Extract value for this key
+ Domain domain = domains.get(recordKeyField);
+ List<String> values = extractStringValues(domain);
+ // First iteration: initialize the queue
+ if (results.isEmpty()) {
+ values.forEach(v -> results.offer(recordKeyField + DEFAULT_COLUMN_VALUE_SEPARATOR + v));
+ }
+ else {
+ int size = results.size();
+ for (int j = 0; j < size; j++) {
+ String currentEntry = results.poll();
+
+ // Generate new combinations by appending keyParts to existing keyParts
+ for (String v : values) {
+ String newKeyPart = recordKeyField + DEFAULT_COLUMN_VALUE_SEPARATOR + v;
+ String newEntry = currentEntry + DEFAULT_RECORD_KEY_PARTS_SEPARATOR + newKeyPart;
+ results.offer(newEntry);
+ }
+ }
+ }
+ }
+ return results.stream().toList();
+ }
+ }
+
+ /**
+ * Extract string values from a domain, handle EQUAL and IN domains only.
+ * Note: Actual implementation depends on your Domain class structure.
+ */
+ private static List<String> extractStringValues(Domain domain)
+ {
+ List<String> values = new ArrayList<>();
+
+ if (domain.isSingleValue()) {
+ // Handle EQUAL condition (single value domain)
+ Object value = domain.getSingleValue();
+ values.add(convertToString(value));
+ }
+ else if (domain.getValues().isDiscreteSet()) {
+ // Handle IN condition (set of discrete values)
+ for (Object value : domain.getValues().getDiscreteSet()) {
+ values.add(convertToString(value));
+ }
+ }
+ return values;
+ }
+
+ private static String convertToString(Object value)
+ {
+ if (value instanceof Slice) {
+ return ((Slice) value).toStringUtf8();
+ }
+ else {
+ return value.toString();
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java
new file mode 100644
index 000000000000..d078c9e6e9f6
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiSecondaryIndexSupport.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.airlift.units.Duration;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.TupleDomainUtils;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieIndexDefinition;
+import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static io.trino.plugin.hudi.HudiSessionProperties.getSecondaryIndexWaitTimeout;
+import static io.trino.plugin.hudi.query.index.HudiRecordLevelIndexSupport.constructRecordKeys;
+import static io.trino.plugin.hudi.query.index.HudiRecordLevelIndexSupport.extractPredicatesForColumns;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+public class HudiSecondaryIndexSupport
+ extends HudiBaseIndexSupport
+{
+ private static final Logger log = Logger.get(HudiSecondaryIndexSupport.class);
+ private final CompletableFuture<Optional<Set<String>>> relevantFileIdsFuture;
+ private final Duration secondaryIndexWaitTimeout;
+ private final long futureStartTimeMs;
+
+ public HudiSecondaryIndexSupport(ConnectorSession session, SchemaTableName schemaTableName, Lazy<HoodieTableMetaClient> lazyMetaClient, Lazy<HoodieTableMetadata> lazyTableMetadata, TupleDomain<HiveColumnHandle> regularColumnPredicates)
+ {
+ super(log, schemaTableName, lazyMetaClient);
+ this.secondaryIndexWaitTimeout = getSecondaryIndexWaitTimeout(session);
+ TupleDomain<String> regularPredicatesTransformed = regularColumnPredicates.transformKeys(HiveColumnHandle::getName);
+ this.relevantFileIdsFuture = CompletableFuture.supplyAsync(() -> {
+ HoodieTimer timer = HoodieTimer.start();
+ if (regularColumnPredicates.isAll() || lazyMetaClient.get().getIndexMetadata().isEmpty()) {
+ log.debug("Predicates cover all data, skipping secondary index lookup.");
+ return Optional.empty();
+ }
+
+ Optional<Map.Entry<String, HoodieIndexDefinition>> firstApplicableIndex = findFirstApplicableSecondaryIndex(regularPredicatesTransformed);
+ if (firstApplicableIndex.isEmpty()) {
+ log.debug("Took %s ms but no secondary index definition found matching the query's referenced columns for table %s",
+ timer.endTimer(), schemaTableName);
+ return Optional.empty();
+ }
+
+ Map.Entry<String, HoodieIndexDefinition> applicableIndexEntry = firstApplicableIndex.get();
+ String indexName = applicableIndexEntry.getKey();
+ // `indexedColumns` should only contain one element as secondary indices only support one column
+ List<String> indexedColumns = applicableIndexEntry.getValue().getSourceFields();
+ log.debug(String.format("Using secondary index '%s' on columns %s for pruning.", indexName, indexedColumns));
+ TupleDomain<String> indexPredicates = extractPredicatesForColumns(regularPredicatesTransformed, indexedColumns);
+
+ List<String> secondaryKeys = constructRecordKeys(indexPredicates, indexedColumns);
+ if (secondaryKeys.isEmpty()) {
+ log.warn("Took %s ms, but could not construct secondary keys for index '%s' from predicates. Skipping pruning for table %s",
+ timer.endTimer(), indexName, schemaTableName);
+ return Optional.empty();
+ }
+ log.debug(String.format("Constructed %d secondary keys for index lookup.", secondaryKeys.size()));
+
+ // Perform index lookup in metadataTable
+ // TODO: document here what this map is keyed by
+ Map<String, HoodieRecordGlobalLocation> recordKeyLocationsMap = lazyTableMetadata.get().readSecondaryIndex(secondaryKeys, indexName);
+ if (recordKeyLocationsMap.isEmpty()) {
+ log.debug("Took %s ms, but secondary index lookup returned no locations for the given keys for table %s", timer.endTimer(), schemaTableName);
+ // Return all original fileSlices
+ return Optional.empty();
+ }
+
+ // Collect fileIds for pruning
+ Set<String> relevantFileIds = recordKeyLocationsMap.values().stream()
+ .map(HoodieRecordGlobalLocation::getFileId)
+ .collect(Collectors.toSet());
+ log.debug(String.format("Secondary index lookup identified %d relevant file IDs.", relevantFileIds.size()));
+
+ return Optional.of(relevantFileIds);
+ });
+
+ this.futureStartTimeMs = System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean shouldSkipFileSlice(FileSlice slice)
+ {
+ try {
+ if (relevantFileIdsFuture.isDone()) {
+ Optional<Set<String>> relevantFileIds = relevantFileIdsFuture.get();
+ return relevantFileIds.map(fileIds -> !fileIds.contains(slice.getFileId())).orElse(false);
+ }
+
+ long elapsedMs = System.currentTimeMillis() - futureStartTimeMs;
+ if (elapsedMs > secondaryIndexWaitTimeout.toMillis()) {
+ // Took too long; skip decision
+ return false;
+ }
+
+ long remainingMs = Math.max(0, secondaryIndexWaitTimeout.toMillis() - elapsedMs);
+ Optional<Set<String>> relevantFileIds = relevantFileIdsFuture.get(remainingMs, MILLISECONDS);
+ return relevantFileIds.map(fileIds -> !fileIds.contains(slice.getFileId())).orElse(false);
+ }
+ catch (TimeoutException | InterruptedException | ExecutionException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Determines whether secondary index (SI) should be used based on the given tuple domain and index definitions.
+ * <p>
+ * This method first filters out the secondary index definitions from the provided map of index definitions.
+ * It then checks if there are any secondary indices defined. If no secondary indices are found, it returns {@code false}.
+ * <p>
+ * For each secondary index definition, the method verifies two conditions:
+ * <ol>
+ * <li>All fields referenced in the tuple domain must be part of the source fields of the secondary index.</li>
+ * <li>The predicates on these fields must be either of type IN or EQUAL.</li>
+ * </ol>
+ * <p>
+ * If at least one secondary index definition meets these conditions, the method returns {@code true}; otherwise,
+ * it returns {@code false}.
+ *
+ * @param tupleDomain the domain representing the constraints on the columns
+ * HoodieIndexDefinition object
+ * @return {@code true} if at least one secondary index can be used based on the given tuple domain; otherwise,
+ * {@code false}
+ */
+ @Override
+ public boolean canApply(TupleDomain<String> tupleDomain)
+ {
+ if (!isIndexSupportAvailable()) {
+ log.debug("Secondary Index partition is not enabled in metadata.");
+ return false;
+ }
+
+ Map<String, HoodieIndexDefinition> secondaryIndexDefinitions = getApplicableIndexDefinitions(tupleDomain, true);
+ if (secondaryIndexDefinitions.isEmpty()) {
+ log.debug("No applicable secondary index definitions found.");
+ return false;
+ }
+
+ boolean atLeastOneIndexUsable = secondaryIndexDefinitions.values().stream()
+ .anyMatch(indexDef -> {
+ List<String> sourceFields = indexDef.getSourceFields();
+ // Predicates referencing columns with secondary index needs to be IN or EQUAL only
+ boolean usable = TupleDomainUtils.areAllFieldsReferenced(tupleDomain, sourceFields)
+ && TupleDomainUtils.areDomainsInOrEqualOnly(tupleDomain, sourceFields);
+ if (log.isDebugEnabled() && usable) {
+ log.debug(String.format("Secondary index '%s' on fields '%s' is usable for the query.", indexDef.getIndexName(), sourceFields));
+ }
+ return usable;
+ });
+ if (!atLeastOneIndexUsable) {
+ log.debug("Although secondary indexes exist, none match the required fields and predicate types (IN/EQUAL) for the query.");
+ }
+ return atLeastOneIndexUsable;
+ }
+
+ private boolean isIndexSupportAvailable()
+ {
+ // Filter out definitions that are secondary indices
+ Map<String, HoodieIndexDefinition> secondaryIndexDefinitions = getAllIndexDefinitions()
+ .entrySet().stream()
+ .filter(e -> e.getKey().contains(HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX))
+ .collect(Collectors.toMap(e -> e.getValue().getIndexName(),
+ Map.Entry::getValue));
+ return !secondaryIndexDefinitions.isEmpty();
+ }
+
+ private Map<String, HoodieIndexDefinition> getApplicableIndexDefinitions(TupleDomain<String> tupleDomain, boolean checkPredicateCompatibility)
+ {
+ Map<String, HoodieIndexDefinition> allDefinitions = getAllIndexDefinitions();
+ if (allDefinitions.isEmpty()) {
+ return Map.of();
+ }
+ // Filter out definitions that are secondary indices
+ return allDefinitions.entrySet().stream()
+ .filter(entry -> entry.getKey().contains(HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX))
+ .filter(entry -> {
+ if (!checkPredicateCompatibility) {
+ return true;
+ }
+ // Perform additional compatibility checks
+ List<String> sourceFields = entry.getValue().getSourceFields();
+ return TupleDomainUtils.areAllFieldsReferenced(tupleDomain, sourceFields)
+ && TupleDomainUtils.areDomainsInOrEqualOnly(tupleDomain, sourceFields);
+ })
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ private Optional<Map.Entry<String, HoodieIndexDefinition>> findFirstApplicableSecondaryIndex(TupleDomain<String> queryPredicates)
+ {
+ // Predicate checks would have already been done, skip predicate checks here
+ Map<String, HoodieIndexDefinition> secondaryIndexDefinitions = getApplicableIndexDefinitions(queryPredicates, false);
+ if (queryPredicates.getDomains().isEmpty()) {
+ return Optional.empty();
+ }
+ List<String> queryReferencedColumns = List.copyOf(queryPredicates.getDomains().get().keySet());
+ return secondaryIndexDefinitions.entrySet().stream()
+ .filter(entry -> {
+ List<String> sourceFields = entry.getValue().getSourceFields();
+ // Only filter for sourceFields that match the predicates
+ return !sourceFields.isEmpty() && queryReferencedColumns.contains(sourceFields.getFirst());
+ })
+ .findFirst();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java
new file mode 100644
index 000000000000..ab78e7662eb2
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/IndexSupportFactory.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.query.index;
+
+import io.airlift.log.Logger;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.SchemaTableName;
+import io.trino.spi.predicate.TupleDomain;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.util.Lazy;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static io.trino.plugin.hudi.HudiSessionProperties.isColumnStatsIndexEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isNoOpIndexEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isPartitionStatsIndexEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isRecordLevelIndexEnabled;
+import static io.trino.plugin.hudi.HudiSessionProperties.isSecondaryIndexEnabled;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Factory to create the appropriate HudiIndexSupport strategy based on:
+ * 1. Query predicates
+ * 2. Available table indexes
+ * 3. Configuration flags
+ */
+public class IndexSupportFactory
+{
+ private static final Logger log = Logger.get(IndexSupportFactory.class);
+
+ private IndexSupportFactory() {}
+
+ /**
+ * Creates the most suitable HudiIndexSupport strategy, considering configuration.
+ * Uses Supplier-based lazy instantiation combined with config checks.
+ *
+ * @param schemaTableName The table schema and name
+ * @param lazyMetaClient The Hudi table metadata client that is lazily instantiated.
+ * @param tupleDomain The query predicates.
+ * @param session Session containing session properties, which is required to control index behaviours for testing/debugging
+ * @return An Optional containing the chosen HudiIndexSupport strategy, or empty if none are applicable or enabled.
+ */
+ public static Optional<HudiIndexSupport> createIndexSupport(
+ SchemaTableName schemaTableName,
+ Lazy<HoodieTableMetaClient> lazyMetaClient,
+ Lazy<HoodieTableMetadata> lazyTableMetadata,
+ TupleDomain<HiveColumnHandle> tupleDomain,
+ ConnectorSession session)
+ {
+ // Define strategies as Suppliers paired with their config (isEnabled) flag
+ // IMPORTANT: Order of strategy here determines which index implementation is preferred first
+ List<StrategyProvider> strategyProviders = List.of(
+ new StrategyProvider(
+ () -> isRecordLevelIndexEnabled(session),
+ () -> new HudiRecordLevelIndexSupport(session, schemaTableName, lazyMetaClient, lazyTableMetadata, tupleDomain)),
+ new StrategyProvider(
+ () -> isSecondaryIndexEnabled(session),
+ () -> new HudiSecondaryIndexSupport(session, schemaTableName, lazyMetaClient, lazyTableMetadata, tupleDomain)),
+ new StrategyProvider(
+ () -> isColumnStatsIndexEnabled(session),
+ () -> new HudiColumnStatsIndexSupport(session, schemaTableName, lazyMetaClient, lazyTableMetadata, tupleDomain)),
+ new StrategyProvider(
+ () -> isNoOpIndexEnabled(session),
+ () -> new HudiNoOpIndexSupport(schemaTableName, lazyMetaClient)));
+
+ TupleDomain<String> transformedTupleDomain = tupleDomain.transformKeys(HiveColumnHandle::getName);
+ for (StrategyProvider provider : strategyProviders) {
+ // Check if the strategy is enabled via config before instantiating
+ if (provider.isEnabled()) {
+ HudiIndexSupport strategy = provider.getStrategy();
+ String strategyName = strategy.getClass().getSimpleName(); // Get name for logging
+
+ // Check if the instantiated strategy is applicable
+ if (strategy.canApply(transformedTupleDomain)) {
+ log.debug(String.format("Selected %s strategy (Enabled & Applicable).", strategyName));
+ return Optional.of(strategy);
+ }
+ else {
+ log.debug(String.format("%s is enabled but not applicable for this query.", strategyName));
+ // Strategy object becomes eligible for GC here, acceptable penalty as the object is lightweight
+ }
+ }
+ else {
+ log.debug(String.format("Strategy associated with supplier %s is disabled by configuration.", provider.supplier.get().getClass().getSimpleName()));
+ }
+ }
+
+ log.debug("No suitable and enabled index support strategy found to be applicable.");
+ return Optional.empty();
+ }
+
+ public static Optional<HudiPartitionStatsIndexSupport> createPartitionStatsIndexSupport(
+ SchemaTableName schemaTableName,
+ Lazy<HoodieTableMetaClient> lazyMetaClient,
+ Lazy<HoodieTableMetadata> lazyTableMetadata,
+ TupleDomain<HiveColumnHandle> tupleDomain,
+ ConnectorSession session)
+ {
+ StrategyProvider partitionStatsStrategy = new StrategyProvider(
+ () -> isPartitionStatsIndexEnabled(session), () -> new HudiPartitionStatsIndexSupport(session, schemaTableName, lazyMetaClient, lazyTableMetadata, tupleDomain));
+
+ TupleDomain<String> transformedTupleDomain = tupleDomain.transformKeys(HiveColumnHandle::getName);
+ if (partitionStatsStrategy.isEnabled() && partitionStatsStrategy.getStrategy().canApply(transformedTupleDomain)) {
+ return Optional.of((HudiPartitionStatsIndexSupport) partitionStatsStrategy.getStrategy());
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * Helper class to pair the configuration check with the strategy supplier to allow for lazy initialization.
+ */
+ private static class StrategyProvider
+ {
+ private final Supplier<Boolean> isEnabled;
+ private final Supplier<HudiIndexSupport> supplier;
+
+ StrategyProvider(Supplier<Boolean> isEnabled, Supplier<HudiIndexSupport> supplier)
+ {
+ this.isEnabled = requireNonNull(isEnabled);
+ this.supplier = requireNonNull(supplier);
+ }
+
+ boolean isEnabled()
+ {
+ return isEnabled.get();
+ }
+
+ HudiIndexSupport getStrategy()
+ {
+ return supplier.get();
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
new file mode 100644
index 000000000000..36c5342920f4
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoReaderContext.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.reader;
+
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.util.HudiAvroSerializer;
+import io.trino.plugin.hudi.util.SynthesizedColumnHandler;
+import io.trino.spi.Page;
+import io.trino.spi.connector.ConnectorPageSource;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericData.Record;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.common.config.RecordMergeMode;
+import org.apache.hudi.common.engine.HoodieReaderContext;
+import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
+import org.apache.hudi.common.model.HoodieAvroRecordMerger;
+import org.apache.hudi.common.model.HoodieEmptyRecord;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordMerger;
+import org.apache.hudi.common.table.read.BufferedRecord;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.ClosableIterator;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+
+public class HudiTrinoReaderContext
+ extends HoodieReaderContext<IndexedRecord>
+{
+ ConnectorPageSource pageSource;
+ private final HudiAvroSerializer avroSerializer;
+ Map<String, Integer> colToPosMap;
+ List<HiveColumnHandle> dataHandles;
+ List<HiveColumnHandle> columnHandles;
+
+ public HudiTrinoReaderContext(
+ ConnectorPageSource pageSource,
+ List<HiveColumnHandle> dataHandles,
+ List<HiveColumnHandle> columnHandles,
+ SynthesizedColumnHandler synthesizedColumnHandler)
+ {
+ this.pageSource = pageSource;
+ this.avroSerializer = new HudiAvroSerializer(columnHandles, synthesizedColumnHandler);
+ this.dataHandles = dataHandles;
+ this.columnHandles = columnHandles;
+ this.colToPosMap = new HashMap<>();
+ for (int i = 0; i < columnHandles.size(); i++) {
+ HiveColumnHandle handle = columnHandles.get(i);
+ colToPosMap.put(handle.getBaseColumnName(), i);
+ }
+ }
+
+ @Override
+ public ClosableIterator<IndexedRecord> getFileRecordIterator(
+ StoragePath storagePath,
+ long start,
+ long length,
+ Schema dataSchema,
+ Schema requiredSchema,
+ HoodieStorage storage)
+ {
+ return new ClosableIterator<>()
+ {
+ private Page currentPage;
+ private int currentPosition;
+
+ @Override
+ public void close()
+ {
+ try {
+ pageSource.close();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ // If all records in the current page are consume, try to get next page
+ if (currentPage == null || currentPosition >= currentPage.getPositionCount()) {
+ if (pageSource.isFinished()) {
+ return false;
+ }
+
+ // Get next page and reset currentPosition
+ currentPage = pageSource.getNextPage();
+ currentPosition = 0;
+
+ // If no more pages are available
+ return currentPage != null;
+ }
+
+ return true;
+ }
+
+ @Override
+ public IndexedRecord next()
+ {
+ if (!hasNext()) {
+ // TODO: This can probably be removed or ignored, added this as a sanity check
+ throw new RuntimeException("No more records in the iterator");
+ }
+
+ IndexedRecord record = avroSerializer.serialize(currentPage, currentPosition);
+ currentPosition++;
+ return record;
+ }
+ };
+ }
+
+ @Override
+ public IndexedRecord convertAvroRecord(IndexedRecord record)
+ {
+ return record;
+ }
+
+ @Override
+ public GenericRecord convertToAvroRecord(IndexedRecord record, Schema schema)
+ {
+ GenericRecord ret = new GenericData.Record(schema);
+ for (Schema.Field field : schema.getFields()) {
+ ret.put(field.name(), record.get(field.pos()));
+ }
+ return ret;
+ }
+
+ @Override
+ public Option<HoodieRecordMerger> getRecordMerger(RecordMergeMode mergeMode, String mergeStrategyId, String mergeImplClasses)
+ {
+ return Option.of(HoodieAvroRecordMerger.INSTANCE);
+ }
+
+ @Override
+ public Object getValue(IndexedRecord record, Schema schema, String fieldName)
+ {
+ if (colToPosMap.containsKey(fieldName)) {
+ return record.get(colToPosMap.get(fieldName));
+ }
+ else {
+ // record doesn't have the queried field, return null
+ return null;
+ }
+ }
+
+ @Override
+ public IndexedRecord seal(IndexedRecord record)
+ {
+ // TODO: this can rely on colToPos map directly instead of schema
+ Schema schema = record.getSchema();
+ IndexedRecord newRecord = new Record(schema);
+ List<Schema.Field> fields = schema.getFields();
+ for (Schema.Field field : fields) {
+ int pos = schema.getField(field.name()).pos();
+ newRecord.put(pos, record.get(pos));
+ }
+ return newRecord;
+ }
+
+ @Override
+ public IndexedRecord toBinaryRow(Schema schema, IndexedRecord record)
+ {
+ return record;
+ }
+
+ @Override
+ public ClosableIterator<IndexedRecord> mergeBootstrapReaders(
+ ClosableIterator closableIterator, Schema schema,
+ ClosableIterator closableIterator1, Schema schema1)
+ {
+ return null;
+ }
+
+ @Override
+ public UnaryOperator<IndexedRecord> projectRecord(
+ Schema from,
+ Schema to,
+ Map<String, String> renamedColumns)
+ {
+ List<Schema.Field> toFields = to.getFields();
+ int[] projection = new int[toFields.size()];
+ for (int i = 0; i < projection.length; i++) {
+ projection[i] = from.getField(toFields.get(i).name()).pos();
+ }
+
+ return fromRecord -> {
+ IndexedRecord toRecord = new Record(to);
+ for (int i = 0; i < projection.length; i++) {
+ toRecord.put(i, fromRecord.get(projection[i]));
+ }
+ return toRecord;
+ };
+ }
+
+ @Override
+ public HoodieRecord<IndexedRecord> constructHoodieRecord(
+ BufferedRecord<IndexedRecord> bufferedRecord)
+ {
+ if (bufferedRecord.isDelete()) {
+ return new HoodieEmptyRecord<>(
+ new HoodieKey(bufferedRecord.getRecordKey(), null),
+ HoodieRecord.HoodieRecordType.AVRO);
+ }
+
+ return new HoodieAvroIndexedRecord(bufferedRecord.getRecord());
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java
new file mode 100644
index 000000000000..7fa0f71399ed
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/reader/HudiTrinoRecord.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.reader;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodieOperation;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.MetadataValues;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.keygen.BaseKeyGenerator;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+
+public class HudiTrinoRecord
+ extends HoodieRecord<IndexedRecord>
+{
+ public HudiTrinoRecord()
+ {
+ }
+
+ @Override
+ public HoodieRecord<IndexedRecord> newInstance()
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord<IndexedRecord> newInstance(HoodieKey hoodieKey, HoodieOperation hoodieOperation)
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord<IndexedRecord> newInstance(HoodieKey hoodieKey)
+ {
+ return null;
+ }
+
+ @Override
+ public Comparable<?> doGetOrderingValue(Schema schema, Properties properties)
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecordType getRecordType()
+ {
+ return null;
+ }
+
+ @Override
+ public String getRecordKey(Schema schema, Option<BaseKeyGenerator> option)
+ {
+ return "";
+ }
+
+ @Override
+ public String getRecordKey(Schema schema, String s)
+ {
+ return "";
+ }
+
+ @Override
+ protected void writeRecordPayload(IndexedRecord page, Kryo kryo, Output output)
+ {
+ }
+
+ @Override
+ protected IndexedRecord readRecordPayload(Kryo kryo, Input input)
+ {
+ return null;
+ }
+
+ @Override
+ public Object[] getColumnValues(Schema schema, String[] strings, boolean b)
+ {
+ return new Object[0];
+ }
+
+ @Override
+ public HoodieRecord joinWith(HoodieRecord hoodieRecord, Schema schema)
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord prependMetaFields(Schema schema, Schema schema1,
+ MetadataValues metadataValues, Properties properties)
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord rewriteRecordWithNewSchema(Schema schema, Properties properties,
+ Schema schema1, Map<String, String> map)
+ {
+ return null;
+ }
+
+ @Override
+ public boolean isDelete(Schema schema, Properties properties)
+ throws IOException
+ {
+ return false;
+ }
+
+ @Override
+ public boolean shouldIgnore(Schema schema, Properties properties)
+ throws IOException
+ {
+ return false;
+ }
+
+ @Override
+ public HoodieRecord<IndexedRecord> copy()
+ {
+ return null;
+ }
+
+ @Override
+ public Option<Map<String, String>> getMetadata()
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord wrapIntoHoodieRecordPayloadWithParams(Schema schema, Properties properties,
+ Option<Pair<String, String>> option, Boolean aBoolean, Option<String> option1,
+ Boolean aBoolean1, Option<Schema> option2)
+ throws IOException
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord wrapIntoHoodieRecordPayloadWithKeyGen(Schema schema, Properties properties,
+ Option<BaseKeyGenerator> option)
+ {
+ return null;
+ }
+
+ @Override
+ public HoodieRecord truncateRecordKey(Schema schema, Properties properties, String s)
+ throws IOException
+ {
+ return null;
+ }
+
+ @Override
+ public Option<HoodieAvroIndexedRecord> toIndexedRecord(Schema schema, Properties properties)
+ throws IOException
+ {
+ return null;
+ }
+
+ @Override
+ public ByteArrayOutputStream getAvroBytes(Schema schema, Properties properties)
+ throws IOException
+ {
+ return null;
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java
new file mode 100644
index 000000000000..56c89ea92634
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiBackgroundSplitLoader.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.split;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import io.airlift.log.Logger;
+import io.trino.metastore.Partition;
+import io.trino.plugin.hive.util.AsyncQueue;
+import io.trino.plugin.hudi.HudiTableHandle;
+import io.trino.plugin.hudi.partition.HudiPartitionInfoLoader;
+import io.trino.plugin.hudi.query.HudiDirectoryLister;
+import io.trino.plugin.hudi.query.index.HudiPartitionStatsIndexSupport;
+import io.trino.plugin.hudi.query.index.IndexSupportFactory;
+import io.trino.spi.TrinoException;
+import io.trino.spi.connector.ConnectorSession;
+import io.trino.spi.connector.ConnectorSplit;
+import io.trino.spi.connector.SchemaTableName;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.util.Lazy;
+
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+import static io.airlift.concurrent.MoreFutures.addExceptionCallback;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_CANNOT_OPEN_SPLIT;
+import static io.trino.plugin.hudi.HudiSessionProperties.getSplitGeneratorParallelism;
+import static io.trino.plugin.hudi.HudiSessionProperties.getTargetSplitSize;
+import static java.util.Objects.requireNonNull;
+
+public class HudiBackgroundSplitLoader
+ implements Runnable
+{
+ private static final Logger log = Logger.get(HudiBackgroundSplitLoader.class);
+ private final HudiTableHandle tableHandle;
+ private final HudiDirectoryLister hudiDirectoryLister;
+ private final AsyncQueue<ConnectorSplit> asyncQueue;
+ private final Executor splitGeneratorExecutor;
+ private final int splitGeneratorNumThreads;
+ private final HudiSplitFactory hudiSplitFactory;
+ private final Lazy<List<String>> lazyPartitions;
+ private final Consumer<Throwable> errorListener;
+ private final boolean enableMetadataTable;
+ private final Optional<HudiPartitionStatsIndexSupport> partitionIndexSupportOpt;
+
+ public HudiBackgroundSplitLoader(
+ ConnectorSession session,
+ HudiTableHandle tableHandle,
+ HudiDirectoryLister hudiDirectoryLister,
+ AsyncQueue<ConnectorSplit> asyncQueue,
+ Executor splitGeneratorExecutor,
+ HudiSplitWeightProvider hudiSplitWeightProvider,
+ Lazy<Map<String, Partition>> lazyPartitionMap,
+ boolean enableMetadataTable,
+ Lazy<HoodieTableMetadata> lazyTableMetadata,
+ Consumer<Throwable> errorListener)
+ {
+ this.tableHandle = requireNonNull(tableHandle, "tableHandle is null");
+ this.hudiDirectoryLister = requireNonNull(hudiDirectoryLister, "hudiDirectoryLister is null");
+ this.asyncQueue = requireNonNull(asyncQueue, "asyncQueue is null");
+ this.splitGeneratorExecutor = requireNonNull(splitGeneratorExecutor, "splitGeneratorExecutorService is null");
+ this.splitGeneratorNumThreads = getSplitGeneratorParallelism(session);
+ this.hudiSplitFactory = new HudiSplitFactory(tableHandle, hudiSplitWeightProvider, getTargetSplitSize(session));
+ this.lazyPartitions = Lazy.lazily(() -> requireNonNull(lazyPartitionMap, "partitions is null").get().keySet().stream().toList());
+ this.enableMetadataTable = enableMetadataTable;
+ this.errorListener = requireNonNull(errorListener, "errorListener is null");
+ SchemaTableName schemaTableName = tableHandle.getSchemaTableName();
+ this.partitionIndexSupportOpt = enableMetadataTable ?
+ IndexSupportFactory.createPartitionStatsIndexSupport(schemaTableName, Lazy.lazily(tableHandle::getMetaClient), lazyTableMetadata, tableHandle.getRegularPredicates(), session) : Optional.empty();
+ }
+
+ @Override
+ public void run()
+ {
+ // Wrap entire logic so that ANY error will be thrown out and not cause program to get stuc
+ try {
+ if (enableMetadataTable) {
+ generateSplits(true);
+ return;
+ }
+
+ // Fallback to partition pruning generator
+ generateSplits(false);
+ }
+ catch (Exception e) {
+ errorListener.accept(e);
+ }
+ }
+
+ private void generateSplits(boolean useIndex)
+ {
+ // Attempt to apply partition pruning using partition stats index
+ Optional<List<String>> effectivePartitionsOpt = useIndex && partitionIndexSupportOpt.isPresent() ?
+ partitionIndexSupportOpt.get().prunePartitions(lazyPartitions.get()) : Optional.empty();
+
+ Deque<String> partitionQueue = new ConcurrentLinkedDeque<>(effectivePartitionsOpt.orElse(lazyPartitions.get()));
+ List<HudiPartitionInfoLoader> splitGenerators = new ArrayList<>();
+ List<ListenableFuture<Void>> futures = new ArrayList<>();
+
+ for (int i = 0; i < splitGeneratorNumThreads; i++) {
+ HudiPartitionInfoLoader generator = new HudiPartitionInfoLoader(hudiDirectoryLister, tableHandle.getLatestCommitTime(), hudiSplitFactory,
+ asyncQueue, partitionQueue, useIndex);
+ splitGenerators.add(generator);
+ ListenableFuture<Void> future = Futures.submit(generator, splitGeneratorExecutor);
+ addExceptionCallback(future, errorListener);
+ futures.add(future);
+ }
+
+ // Signal all generators to stop once partition queue is drained
+ splitGenerators.forEach(HudiPartitionInfoLoader::stopRunning);
+
+ log.info("Wait for partition pruning split generation to finish on table %s.%s", tableHandle.getSchemaName(), tableHandle.getTableName());
+ try {
+ Futures.whenAllComplete(futures)
+ .run(asyncQueue::finish, directExecutor())
+ .get();
+ log.info("Partition pruning split generation finished on table %s.%s", tableHandle.getSchemaName(), tableHandle.getTableName());
+ }
+ catch (InterruptedException | ExecutionException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new TrinoException(HUDI_CANNOT_OPEN_SPLIT, "Error generating Hudi split", e);
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
new file mode 100644
index 000000000000..872f56cb352d
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitFactory.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.split;
+
+import com.google.common.collect.ImmutableList;
+import io.airlift.units.DataSize;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hudi.HudiSplit;
+import io.trino.plugin.hudi.HudiTableHandle;
+import io.trino.plugin.hudi.file.HudiBaseFile;
+import io.trino.plugin.hudi.file.HudiLogFile;
+import io.trino.spi.TrinoException;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.util.Option;
+
+import java.util.Collections;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static io.trino.plugin.hudi.HudiErrorCode.HUDI_FILESYSTEM_ERROR;
+import static java.lang.String.format;
+import static java.util.Objects.requireNonNull;
+
+public class HudiSplitFactory
+{
+ private static final double SPLIT_SLOP = 1.1; // 10% slop/overflow allowed in bytes per split while generating splits
+
+ private final HudiTableHandle hudiTableHandle;
+ private final HudiSplitWeightProvider hudiSplitWeightProvider;
+ private final DataSize targetSplitSize;
+
+ public HudiSplitFactory(
+ HudiTableHandle hudiTableHandle,
+ HudiSplitWeightProvider hudiSplitWeightProvider,
+ DataSize targetSplitSize)
+ {
+ this.hudiTableHandle = requireNonNull(hudiTableHandle, "hudiTableHandle is null");
+ this.hudiSplitWeightProvider = requireNonNull(hudiSplitWeightProvider, "hudiSplitWeightProvider is null");
+ this.targetSplitSize = requireNonNull(targetSplitSize, "targetSplitSize is null");
+ }
+
+ public List<HudiSplit> createSplits(List<HivePartitionKey> partitionKeys, FileSlice fileSlice, String commitTime)
+ {
+ return createHudiSplits(hudiTableHandle, partitionKeys, fileSlice, commitTime, hudiSplitWeightProvider, targetSplitSize);
+ }
+
+ public static List<HudiSplit> createHudiSplits(
+ HudiTableHandle hudiTableHandle,
+ List<HivePartitionKey> partitionKeys,
+ FileSlice fileSlice,
+ String commitTime,
+ HudiSplitWeightProvider hudiSplitWeightProvider,
+ DataSize targetSplitSize)
+ {
+ if (fileSlice.isEmpty()) {
+ throw new TrinoException(HUDI_FILESYSTEM_ERROR, format("Not a valid file slice: %s", fileSlice.toString()));
+ }
+
+ // Handle MERGE_ON_READ tables to be read in read_optimized mode
+ // IMPORTANT: These tables will have a COPY_ON_WRITE table type due to how `HudiTableTypeUtils#fromInputFormat`
+ if (fileSlice.getLogFiles().findAny().isEmpty()
+ || hudiTableHandle.getTableType().equals(HoodieTableType.COPY_ON_WRITE)) {
+ // Base file only
+ checkArgument(fileSlice.getBaseFile().isPresent(),
+ "Hudi base file must exist if there is no log file in the file slice");
+ HoodieBaseFile baseFile = fileSlice.getBaseFile().get();
+ long fileSize = baseFile.getFileSize();
+
+ if (fileSize == 0) {
+ return ImmutableList.of(new HudiSplit(
+ HudiBaseFile.of(baseFile),
+ Collections.emptyList(),
+ commitTime,
+ hudiTableHandle.getRegularPredicates(),
+ partitionKeys,
+ hudiSplitWeightProvider.calculateSplitWeight(fileSize)));
+ }
+
+ ImmutableList.Builder<HudiSplit> splits = ImmutableList.builder();
+ long targetSplitSizeInBytes = Math.max(targetSplitSize.toBytes(), baseFile.getPathInfo().getBlockSize());
+
+ long bytesRemaining = fileSize;
+ while (((double) bytesRemaining) / targetSplitSizeInBytes > SPLIT_SLOP) {
+ splits.add(new HudiSplit(
+ HudiBaseFile.of(baseFile, fileSize - bytesRemaining, targetSplitSizeInBytes),
+ Collections.emptyList(),
+ commitTime,
+ hudiTableHandle.getRegularPredicates(),
+ partitionKeys,
+ hudiSplitWeightProvider.calculateSplitWeight(targetSplitSizeInBytes)));
+ bytesRemaining -= targetSplitSizeInBytes;
+ }
+ if (bytesRemaining > 0) {
+ splits.add(new HudiSplit(
+ HudiBaseFile.of(baseFile, fileSize - bytesRemaining, bytesRemaining),
+ Collections.emptyList(),
+ commitTime,
+ hudiTableHandle.getRegularPredicates(),
+ partitionKeys,
+ hudiSplitWeightProvider.calculateSplitWeight(bytesRemaining)));
+ }
+ return splits.build();
+ }
+
+ // Base and log files
+ Option<HoodieBaseFile> baseFileOption = fileSlice.getBaseFile();
+ return ImmutableList.of(new HudiSplit(
+ baseFileOption.isPresent() ? HudiBaseFile.of(baseFileOption.get()) : null,
+ fileSlice.getLogFiles().map(HudiLogFile::of).toList(),
+ commitTime,
+ hudiTableHandle.getRegularPredicates(),
+ partitionKeys,
+ hudiSplitWeightProvider.calculateSplitWeight(fileSlice.getTotalFileSize())));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java
new file mode 100644
index 000000000000..dd90dd356340
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/HudiSplitWeightProvider.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.split;
+
+import io.trino.spi.SplitWeight;
+
+public interface HudiSplitWeightProvider
+{
+ SplitWeight calculateSplitWeight(long splitSizeInBytes);
+
+ static HudiSplitWeightProvider uniformStandardWeightProvider()
+ {
+ return splitSizeInBytes -> SplitWeight.standard();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java
new file mode 100644
index 000000000000..e94e2d73c373
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/split/SizeBasedSplitWeightProvider.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.split;
+
+import io.airlift.units.DataSize;
+import io.trino.spi.SplitWeight;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.Math.clamp;
+import static java.util.Objects.requireNonNull;
+
+public class SizeBasedSplitWeightProvider
+ implements HudiSplitWeightProvider
+{
+ private final double minimumWeight;
+ private final double standardSplitSizeInBytes;
+
+ public SizeBasedSplitWeightProvider(double minimumWeight, DataSize standardSplitSize)
+ {
+ checkArgument(
+ Double.isFinite(minimumWeight) && minimumWeight > 0 && minimumWeight <= 1,
+ "minimumWeight must be > 0 and <= 1, found: %s", minimumWeight);
+ this.minimumWeight = minimumWeight;
+ long standardSplitSizeInBytesLong = requireNonNull(standardSplitSize, "standardSplitSize is null").toBytes();
+ checkArgument(standardSplitSizeInBytesLong > 0, "standardSplitSize must be > 0, found: %s", standardSplitSize);
+ this.standardSplitSizeInBytes = (double) standardSplitSizeInBytesLong;
+ }
+
+ @Override
+ public SplitWeight calculateSplitWeight(long splitSizeInBytes)
+ {
+ double computedWeight = splitSizeInBytes / standardSplitSizeInBytes;
+ // Clamp the value between the minimum weight and 1.0 (standard weight)
+ return SplitWeight.fromProportion(clamp(computedWeight, minimumWeight, 1.0));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java
new file mode 100644
index 000000000000..f324887a3e7c
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/ForHudiTableStatistics.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.stats;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@BindingAnnotation
+public @interface ForHudiTableStatistics {}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java
new file mode 100644
index 000000000000..04526521b6ae
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/HudiTableStatistics.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.stats;
+
+import io.trino.spi.statistics.TableStatistics;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+
+public record HudiTableStatistics(
+ HoodieInstant latestCommit,
+ TableStatistics tableStatistics)
+{
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java
new file mode 100644
index 000000000000..e87122de0795
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableMetadataReader.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.stats;
+
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieColumnRangeMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.common.util.hash.ColumnIndexID;
+import org.apache.hudi.common.util.hash.FileIndexID;
+import org.apache.hudi.common.util.hash.PartitionIndexID;
+import org.apache.hudi.exception.HoodieMetadataException;
+import org.apache.hudi.metadata.HoodieBackedTableMetadata;
+import org.apache.hudi.metadata.HoodieMetadataMetrics;
+import org.apache.hudi.metadata.HoodieMetadataPayload;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.storage.HoodieStorage;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Reads metadata efficiently from a Hudi metadata table.
+ */
+public class TableMetadataReader
+ extends HoodieBackedTableMetadata
+{
+ TableMetadataReader(HoodieEngineContext engineContext, HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig, String datasetBasePath, boolean reuse)
+ {
+ super(engineContext, storage, metadataConfig, datasetBasePath, reuse);
+ }
+
+ /**
+ * Retrieves column statistics for the specified partition and file names.
+ *
+ * @param partitionNameFileNameList a list of partition and file name pairs for which column statistics are retrieved
+ * @param columnNames a list of column names for which statistics are needed
+ * @return a map from column name to their corresponding {@link HoodieColumnRangeMetadata}
+ * @throws HoodieMetadataException if an error occurs while fetching the column statistics
+ */
+ Map<String, HoodieColumnRangeMetadata> getColumnStats(List<Pair<String, String>> partitionNameFileNameList, List<String> columnNames)
+ throws HoodieMetadataException
+ {
+ return computeFileToColumnStatsMap(computeColumnStatsLookupKeys(partitionNameFileNameList, columnNames));
+ }
+
+ /**
+ * @param partitionNameFileNameList a list of partition and file name pairs for which column stats need to be retrieved
+ * @param columnNames list of column names for which stats are needed
+ * @return a list of column stats keys to look up in the metadata table col_stats partition.
+ */
+ private List<String> computeColumnStatsLookupKeys(
+ final List<Pair<String, String>> partitionNameFileNameList,
+ final List<String> columnNames)
+ {
+ return columnNames.stream()
+ .flatMap(columnName -> partitionNameFileNameList.stream()
+ .map(partitionNameFileNamePair -> HoodieMetadataPayload.getColumnStatsIndexKey(
+ new PartitionIndexID(HoodieTableMetadataUtil.getColumnStatsIndexPartitionIdentifier(partitionNameFileNamePair.getLeft())),
+ new FileIndexID(partitionNameFileNamePair.getRight()),
+ new ColumnIndexID(columnName))))
+ .toList();
+ }
+
+ /**
+ * @param columnStatsLookupKeys a map from column stats key to partition and file name pair
+ * @return a map from column name to merged HoodieMetadataColumnStats
+ */
+ private Map<String, HoodieColumnRangeMetadata> computeFileToColumnStatsMap(List<String> columnStatsLookupKeys)
+ {
+ HoodieTimer timer = HoodieTimer.start();
+ Map<String, HoodieRecord<HoodieMetadataPayload>> hoodieRecords =
+ getRecordsByKeys(columnStatsLookupKeys, MetadataPartitionType.COLUMN_STATS.getPartitionPath());
+ metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR, timer.endTimer()));
+ return hoodieRecords.values().stream()
+ .collect(Collectors.groupingBy(
+ r -> r.getData().getColumnStatMetadata().get().getColumnName(),
+ Collectors.mapping(r -> r.getData().getColumnStatMetadata().get(), Collectors.toList())))
+ .entrySet().stream()
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ e -> {
+ long valueCount = 0L;
+ long nullCount = 0L;
+ long totalSize = 0L;
+ long totalUncompressedSize = 0L;
+ for (HoodieMetadataColumnStats stats : e.getValue()) {
+ valueCount += stats.getValueCount();
+ nullCount += stats.getNullCount();
+ totalSize += stats.getTotalSize();
+ totalUncompressedSize += stats.getTotalUncompressedSize();
+ }
+ return HoodieColumnRangeMetadata.create(
+ "", e.getKey(), null, null, nullCount, valueCount, totalSize, totalUncompressedSize);
+ }));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java
new file mode 100644
index 000000000000..30e03622f577
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/stats/TableStatisticsReader.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.stats;
+
+import com.google.common.collect.ImmutableMap;
+import io.airlift.log.Logger;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hudi.storage.TrinoStorageConfiguration;
+import io.trino.spi.connector.ColumnHandle;
+import io.trino.spi.statistics.ColumnStatistics;
+import io.trino.spi.statistics.Estimate;
+import io.trino.spi.statistics.TableStatistics;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.model.HoodieColumnRangeMetadata;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.util.collection.Pair;
+
+import java.util.List;
+import java.util.Map;
+
+import static io.trino.plugin.hudi.HudiUtil.getFileSystemView;
+
+/**
+ * Reads table statistics of a Hudi table from the metadata table files and column stats partitions.
+ */
+public class TableStatisticsReader
+{
+ private static final Logger log = Logger.get(TableStatisticsReader.class);
+ private final HoodieTableMetaClient metaClient;
+ private final TableMetadataReader tableMetadata;
+ private final HoodieTableFileSystemView fileSystemView;
+
+ private TableStatisticsReader(HoodieTableMetaClient metaClient)
+ {
+ this.metaClient = metaClient;
+ HoodieEngineContext engineContext = new HoodieLocalEngineContext(new TrinoStorageConfiguration());
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build();
+ this.tableMetadata = new TableMetadataReader(
+ engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString(), true);
+ this.fileSystemView = getFileSystemView(tableMetadata, metaClient);
+ }
+
+ public static TableStatisticsReader create(HoodieTableMetaClient metaClient)
+ {
+ return new TableStatisticsReader(metaClient);
+ }
+
+ /**
+ * Retrieves table statistics of a Hudi table based on the latest commit and specified columns.
+ *
+ * @param latestCommit the most recent commit at which to retrieve the statistics
+ * @param columnHandles a list of {@link HiveColumnHandle} representing the columns for which statistics are needed
+ * @return {@link TableStatistics} object containing the statistics of the specified columns, or empty statistics if unable to retrieve
+ */
+ public TableStatistics getTableStatistics(HoodieInstant latestCommit,
+ List<HiveColumnHandle> columnHandles)
+ {
+ List<String> columnNames = columnHandles.stream()
+ .map(HiveColumnHandle::getName)
+ .toList();
+ Map<String, HoodieColumnRangeMetadata> columnStatsMap = getColumnStats(latestCommit, tableMetadata, fileSystemView, columnNames);
+ if (columnStatsMap.isEmpty()) {
+ log.warn("Unable to get column stats from metadata table for table, returning empty table statistics: %s",
+ metaClient.getBasePath());
+ return TableStatistics.empty();
+ }
+ long rowCount = columnStatsMap.values().stream()
+ .map(e -> e.getNullCount() + e.getValueCount())
+ .max(Long::compare)
+ .get();
+ ImmutableMap.Builder<ColumnHandle, ColumnStatistics> columnHandleBuilder = ImmutableMap.builder();
+ for (HiveColumnHandle columnHandle : columnHandles) {
+ HoodieColumnRangeMetadata columnStats = columnStatsMap.get(columnHandle.getName());
+ if (columnStats == null) {
+ log.warn("Unable to get column stats for column %s in table %s",
+ columnHandle.getName(), metaClient.getBasePath());
+ continue;
+ }
+ ColumnStatistics.Builder columnStatisticsBuilder = new ColumnStatistics.Builder();
+ long totalCount = columnStats.getNullCount() + columnStats.getValueCount();
+ columnStatisticsBuilder.setNullsFraction(Estimate.of(
+ columnStats.getNullCount() / (double) totalCount));
+ columnStatisticsBuilder.setDataSize(Estimate.of(columnStats.getTotalUncompressedSize() / (double) totalCount));
+ columnHandleBuilder.put(columnHandle, columnStatisticsBuilder.build());
+ }
+ return new TableStatistics(Estimate.of(rowCount), columnHandleBuilder.buildOrThrow());
+ }
+
+ private static Map<String, HoodieColumnRangeMetadata> getColumnStats(
+ HoodieInstant latestCommit,
+ TableMetadataReader tableMetadata,
+ HoodieTableFileSystemView fileSystemView,
+ List<String> columnNames)
+ {
+ fileSystemView.loadAllPartitions();
+ List<Pair<String, String>> filePaths = fileSystemView.getAllLatestBaseFilesBeforeOrOn(latestCommit.requestedTime())
+ .entrySet()
+ .stream().flatMap(entry -> entry.getValue()
+ .map(baseFile -> Pair.of(entry.getKey(), baseFile.getFileName())))
+ .toList();
+ return tableMetadata.getColumnStats(filePaths, columnNames);
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java
new file mode 100644
index 000000000000..26e1eae088a0
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoInlineStorage.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.storage;
+
+import io.trino.filesystem.TrinoInputStream;
+import io.trino.plugin.hudi.io.InlineSeekableDataInputStream;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.exception.HoodieNotSupportedException;
+import org.apache.hudi.io.SeekableDataInputStream;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathFilter;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.inline.InLineFSUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+
+public class HudiTrinoInlineStorage
+ extends HoodieStorage
+{
+ private final HudiTrinoStorage storage;
+
+ public HudiTrinoInlineStorage(HudiTrinoStorage storage)
+ {
+ super(storage.getConf());
+ this.storage = storage;
+ }
+
+ @Override
+ public HoodieStorage newInstance(StoragePath path, StorageConfiguration<?> storageConf)
+ {
+ if (InLineFSUtils.SCHEME.equals(path.toUri().getScheme())) {
+ return this;
+ }
+ return storage;
+ }
+
+ @Override
+ public String getScheme()
+ {
+ return InLineFSUtils.SCHEME;
+ }
+
+ @Override
+ public int getDefaultBlockSize(StoragePath path)
+ {
+ return storage.getDefaultBlockSize(getFilePathFromInlinePath(path));
+ }
+
+ @Override
+ public int getDefaultBufferSize()
+ {
+ return storage.getDefaultBufferSize();
+ }
+
+ @Override
+ public short getDefaultReplication(StoragePath path)
+ {
+ return storage.getDefaultReplication(getFilePathFromInlinePath(path));
+ }
+
+ @Override
+ public URI getUri()
+ {
+ return URI.create(getScheme());
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean b)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean b, Integer integer, Short aShort, Long aLong)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public InputStream open(StoragePath path)
+ throws IOException
+ {
+ return openSeekable(path, getDefaultBufferSize(), false);
+ }
+
+ @Override
+ public SeekableDataInputStream openSeekable(StoragePath path, int bufferSize, boolean wrapStream)
+ throws IOException
+ {
+ return new InlineSeekableDataInputStream(
+ (TrinoInputStream) storage.open(getFilePathFromInlinePath(path)),
+ InLineFSUtils.startOffset(path),
+ InLineFSUtils.length(path));
+ }
+
+ @Override
+ public OutputStream append(StoragePath path)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public boolean exists(StoragePath path)
+ throws IOException
+ {
+ return storage.exists(getFilePathFromInlinePath(path));
+ }
+
+ @Override
+ public StoragePathInfo getPathInfo(StoragePath path)
+ throws IOException
+ {
+ StoragePathInfo pathInfo = storage.getPathInfo(getFilePathFromInlinePath(path));
+ return new StoragePathInfo(
+ path, InLineFSUtils.length(path), pathInfo.isDirectory(),
+ pathInfo.getBlockReplication(), pathInfo.getBlockSize(),
+ pathInfo.getModificationTime());
+ }
+
+ @Override
+ public boolean createDirectory(StoragePath path)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public List<StoragePathInfo> listDirectEntries(StoragePath path)
+ throws IOException
+ {
+ // This is supposed to be called on a file path only
+ StoragePathInfo pathInfo = getPathInfo(path);
+ ValidationUtils.checkArgument(pathInfo.isFile(),
+ "HudiTrinoInlineStorage#listDirectEntries should only be called on a file path");
+ return Collections.singletonList(pathInfo);
+ }
+
+ @Override
+ public List<StoragePathInfo> listFiles(StoragePath path)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("This API should not be called by Hudi inline storage.");
+ }
+
+ @Override
+ public List<StoragePathInfo> listDirectEntries(StoragePath path, StoragePathFilter storagePathFilter)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("This API should not be called by Hudi inline storage.");
+ }
+
+ @Override
+ public List<StoragePathInfo> globEntries(StoragePath path, StoragePathFilter storagePathFilter)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("This API should not be called by Hudi inline storage.");
+ }
+
+ @Override
+ public boolean rename(StoragePath oldPath, StoragePath newPath)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public boolean deleteDirectory(StoragePath path)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public boolean deleteFile(StoragePath path)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public void setModificationTime(StoragePath path, long modificationTime)
+ throws IOException
+ {
+ throw new HoodieNotSupportedException("Hudi inline storage supports reads only.");
+ }
+
+ @Override
+ public Object getFileSystem()
+ {
+ return storage.getFileSystem();
+ }
+
+ @Override
+ public HoodieStorage getRawStorage()
+ {
+ return this;
+ }
+
+ @Override
+ public void close()
+ throws IOException
+ {
+ }
+
+ private static StoragePath getFilePathFromInlinePath(StoragePath inlinePath)
+ {
+ assertInlineFSPath(inlinePath);
+ String outerFileScheme = inlinePath.getParent().getName();
+ StoragePath basePath = inlinePath.getParent().getParent();
+ ValidationUtils.checkArgument(basePath.toString().contains(":"), "Invalid InLineFS path: " + inlinePath);
+ String pathExceptScheme = basePath.toString().substring(basePath.toString().indexOf(":") + 1);
+ String fullPath = outerFileScheme + ":" + (outerFileScheme.equals("file") || outerFileScheme.equals("local") ? "/" : "") + pathExceptScheme;
+ return new StoragePath(fullPath);
+ }
+
+ private static void assertInlineFSPath(StoragePath inlinePath)
+ {
+ String scheme = inlinePath.toUri().getScheme();
+ ValidationUtils.checkArgument(InLineFSUtils.SCHEME.equals(scheme));
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
new file mode 100644
index 000000000000..48c5409c10d8
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/HudiTrinoStorage.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.storage;
+
+import io.airlift.units.DataSize;
+import io.trino.filesystem.FileEntry;
+import io.trino.filesystem.FileIterator;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoInputFile;
+import io.trino.plugin.hudi.io.TrinoSeekableDataInputStream;
+import org.apache.hudi.io.SeekableDataInputStream;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathFilter;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.inline.InLineFSUtils;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static io.airlift.units.DataSize.Unit.MEGABYTE;
+
+public class HudiTrinoStorage
+ extends HoodieStorage
+{
+ private static final int DEFAULT_BLOCK_SIZE = (int) DataSize.of(32, MEGABYTE).toBytes();
+ private static final int DEFAULT_BUFFER_SIZE = 4096;
+ private static final int DEFAULT_REPLICATION = 1;
+
+ private final TrinoFileSystem fileSystem;
+
+ public HudiTrinoStorage(TrinoFileSystem fileSystem, TrinoStorageConfiguration storageConf)
+ {
+ super(storageConf);
+ this.fileSystem = fileSystem;
+ }
+
+ public static Location convertToLocation(StoragePath path)
+ {
+ return Location.of(path.toString());
+ }
+
+ public static StoragePath convertToPath(Location location)
+ {
+ return new StoragePath(location.toString());
+ }
+
+ public static StoragePathInfo convertToPathInfo(FileEntry fileEntry)
+ {
+ return new StoragePathInfo(
+ convertToPath(fileEntry.location()),
+ fileEntry.length(),
+ false,
+ (short) 0,
+ 0,
+ fileEntry.lastModified().toEpochMilli());
+ }
+
+ @Override
+ public HoodieStorage newInstance(StoragePath path, StorageConfiguration<?> storageConf)
+ {
+ if (InLineFSUtils.SCHEME.equals(path.toUri().getScheme())) {
+ return new HudiTrinoInlineStorage(this);
+ }
+ return this;
+ }
+
+ @Override
+ public String getScheme()
+ {
+ return "file";
+ }
+
+ @Override
+ public int getDefaultBlockSize(StoragePath path)
+ {
+ return DEFAULT_BLOCK_SIZE;
+ }
+
+ @Override
+ public int getDefaultBufferSize()
+ {
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ @Override
+ public short getDefaultReplication(StoragePath path)
+ {
+ return DEFAULT_REPLICATION;
+ }
+
+ @Override
+ public URI getUri()
+ {
+ return URI.create(getScheme());
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean overwrite)
+ throws IOException
+ {
+ return fileSystem.newOutputFile(convertToLocation(path)).create();
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean overwrite, Integer bufferSize, Short replication, Long sizeThreshold)
+ throws IOException
+ {
+ return create(path, overwrite);
+ }
+
+ @Override
+ public InputStream open(StoragePath path)
+ throws IOException
+ {
+ return fileSystem.newInputFile(convertToLocation(path)).newStream();
+ }
+
+ @Override
+ public SeekableDataInputStream openSeekable(StoragePath path, int bufferSize, boolean wrapStream)
+ throws IOException
+ {
+ return new TrinoSeekableDataInputStream(
+ fileSystem.newInputFile(convertToLocation(path)).newStream());
+ }
+
+ @Override
+ public OutputStream append(StoragePath path)
+ throws IOException
+ {
+ throw new UnsupportedOperationException("HudiTrinoStorage does not support append operation.");
+ }
+
+ @Override
+ public boolean exists(StoragePath path)
+ throws IOException
+ {
+ return fileSystem.newInputFile(convertToLocation(path)).exists();
+ }
+
+ @Override
+ public StoragePathInfo getPathInfo(StoragePath path)
+ throws IOException
+ {
+ Location location = convertToLocation(path);
+ Optional<Boolean> result = fileSystem.directoryExists(location);
+ if (result.isPresent() && result.get()) {
+ return new StoragePathInfo(path, 0, true, (short) 0, 0, 0);
+ }
+ TrinoInputFile inputFile = fileSystem.newInputFile(location);
+ if (!inputFile.exists()) {
+ throw new FileNotFoundException("Path " + path + " does not exist");
+ }
+ return new StoragePathInfo(path, inputFile.length(), false, (short) 0, 0, inputFile.lastModified().toEpochMilli());
+ }
+
+ @Override
+ public boolean createDirectory(StoragePath path)
+ throws IOException
+ {
+ fileSystem.createDirectory(convertToLocation(path));
+ return true;
+ }
+
+ @Override
+ public List<StoragePathInfo> listDirectEntries(StoragePath path)
+ throws IOException
+ {
+ // TrinoFileSystem#listFiles lists recursively, we need to limit the result to only the direct children
+ Location location = convertToLocation(path);
+ FileIterator fileIterator = fileSystem.listFiles(location);
+ List<StoragePathInfo> fileList = new ArrayList<>();
+ while (fileIterator.hasNext()) {
+ FileEntry entry = fileIterator.next();
+ if (entry.location().parentDirectory().path().equals(location.path())) {
+ fileList.add(convertToPathInfo(entry));
+ }
+ }
+ return fileList;
+ }
+
+ @Override
+ public List<StoragePathInfo> listFiles(StoragePath path)
+ throws IOException
+ {
+ FileIterator fileIterator = fileSystem.listFiles(convertToLocation(path));
+ List<StoragePathInfo> fileList = new ArrayList<>();
+ while (fileIterator.hasNext()) {
+ fileList.add(convertToPathInfo(fileIterator.next()));
+ }
+ return fileList;
+ }
+
+ @Override
+ public List<StoragePathInfo> listDirectEntries(StoragePath path, StoragePathFilter filter)
+ throws IOException
+ {
+ // TrinoFileSystem#listFiles lists recursively, we need to limit the result to only the direct children
+ Location location = convertToLocation(path);
+ FileIterator fileIterator = fileSystem.listFiles(location);
+ List<StoragePathInfo> fileList = new ArrayList<>();
+ while (fileIterator.hasNext()) {
+ FileEntry entry = fileIterator.next();
+ if (filter.accept(new StoragePath(entry.location().toString()))
+ && entry.location().parentDirectory().path().equals(location.path())) {
+ fileList.add(convertToPathInfo(entry));
+ }
+ }
+ return fileList;
+ }
+
+ @Override
+ public List<StoragePathInfo> globEntries(StoragePath pathPattern, StoragePathFilter filter)
+ throws IOException
+ {
+ throw new UnsupportedOperationException("HudiTrinoStorage does not support globEntries operation.");
+ }
+
+ @Override
+ public boolean rename(StoragePath oldPath, StoragePath newPath)
+ throws IOException
+ {
+ fileSystem.renameFile(convertToLocation(oldPath), convertToLocation(newPath));
+ return true;
+ }
+
+ @Override
+ public boolean deleteDirectory(StoragePath path)
+ throws IOException
+ {
+ fileSystem.deleteDirectory(convertToLocation(path));
+ return true;
+ }
+
+ @Override
+ public boolean deleteFile(StoragePath path)
+ throws IOException
+ {
+ fileSystem.deleteFile(convertToLocation(path));
+ return true;
+ }
+
+ @Override
+ public void setModificationTime(StoragePath path, long modificationTime)
+ throws IOException
+ {
+ Location sameLocation = convertToLocation(path);
+ fileSystem.renameFile(sameLocation, sameLocation);
+ }
+
+ @Override
+ public Object getFileSystem()
+ {
+ return fileSystem;
+ }
+
+ @Override
+ public HoodieStorage getRawStorage()
+ {
+ return this;
+ }
+
+ @Override
+ public void close()
+ throws IOException
+ {
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java
new file mode 100644
index 000000000000..48939ae84c5a
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/storage/TrinoStorageConfiguration.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.storage;
+
+import io.trino.plugin.hudi.io.HudiTrinoIOFactory;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.storage.StorageConfiguration;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hudi.common.config.HoodieStorageConfig.HOODIE_IO_FACTORY_CLASS;
+import static org.apache.hudi.common.config.HoodieStorageConfig.HOODIE_STORAGE_CLASS;
+
+public class TrinoStorageConfiguration
+ extends StorageConfiguration
+{
+ private final Map<String, String> configMap;
+
+ public TrinoStorageConfiguration()
+ {
+ this(getDefaultConfigs());
+ }
+
+ public TrinoStorageConfiguration(Map<String, String> configMap)
+ {
+ this.configMap = configMap;
+ }
+
+ public static Map<String, String> getDefaultConfigs()
+ {
+ Map<String, String> configMap = new HashMap<>();
+ configMap.put(HOODIE_IO_FACTORY_CLASS.key(), HudiTrinoIOFactory.class.getName());
+ configMap.put(HOODIE_STORAGE_CLASS.key(), HudiTrinoStorage.class.getName());
+ return configMap;
+ }
+
+ @Override
+ public StorageConfiguration newInstance()
+ {
+ return new TrinoStorageConfiguration(new HashMap<>(configMap));
+ }
+
+ @Override
+ public Object unwrap()
+ {
+ return configMap;
+ }
+
+ @Override
+ public Object unwrapCopy()
+ {
+ return new HashMap<>(configMap);
+ }
+
+ @Override
+ public void set(String key, String value)
+ {
+ configMap.put(key, value);
+ }
+
+ @Override
+ public Option<String> getString(String key)
+ {
+ return Option.ofNullable(configMap.get(key));
+ }
+
+ @Override
+ public StorageConfiguration getInline()
+ {
+ return newInstance();
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java
new file mode 100644
index 000000000000..91b38f593e19
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java
@@ -0,0 +1,504 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.util;
+
+import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.spi.Page;
+import io.trino.spi.PageBuilder;
+import io.trino.spi.TrinoException;
+import io.trino.spi.block.ArrayBlockBuilder;
+import io.trino.spi.block.BlockBuilder;
+import io.trino.spi.block.Fixed12BlockBuilder;
+import io.trino.spi.block.MapBlockBuilder;
+import io.trino.spi.block.RowBlockBuilder;
+import io.trino.spi.type.ArrayType;
+import io.trino.spi.type.CharType;
+import io.trino.spi.type.DateType;
+import io.trino.spi.type.DecimalType;
+import io.trino.spi.type.Decimals;
+import io.trino.spi.type.Int128;
+import io.trino.spi.type.LongTimestamp;
+import io.trino.spi.type.LongTimestampWithTimeZone;
+import io.trino.spi.type.MapType;
+import io.trino.spi.type.RowType;
+import io.trino.spi.type.SqlDate;
+import io.trino.spi.type.SqlDecimal;
+import io.trino.spi.type.SqlTimestamp;
+import io.trino.spi.type.SqlVarbinary;
+import io.trino.spi.type.Type;
+import io.trino.spi.type.VarbinaryType;
+import io.trino.spi.type.VarcharType;
+import org.apache.avro.Conversions;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.avro.util.Utf8;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Verify.verify;
+import static io.airlift.slice.Slices.utf8Slice;
+import static io.trino.plugin.hudi.HudiUtil.constructSchema;
+import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
+import static io.trino.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE;
+import static io.trino.spi.type.BigintType.BIGINT;
+import static io.trino.spi.type.DateType.DATE;
+import static io.trino.spi.type.Decimals.encodeShortScaledValue;
+import static io.trino.spi.type.Decimals.writeBigDecimal;
+import static io.trino.spi.type.Decimals.writeShortDecimal;
+import static io.trino.spi.type.IntegerType.INTEGER;
+import static io.trino.spi.type.LongTimestampWithTimeZone.fromEpochMillisAndFraction;
+import static io.trino.spi.type.RealType.REAL;
+import static io.trino.spi.type.SmallintType.SMALLINT;
+import static io.trino.spi.type.TimeType.TIME_MICROS;
+import static io.trino.spi.type.TimeZoneKey.UTC_KEY;
+import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS;
+import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MICROS;
+import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND;
+import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND;
+import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND;
+import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND;
+import static io.trino.spi.type.TinyintType.TINYINT;
+import static java.lang.Integer.parseInt;
+import static java.lang.Math.floorDiv;
+import static java.lang.Math.floorMod;
+import static java.lang.Math.toIntExact;
+import static java.lang.String.format;
+import static java.time.ZoneOffset.UTC;
+
+public class HudiAvroSerializer
+{
+ private static final int[] NANO_FACTOR = {
+ -1, // 0, no need to multiply
+ 100_000_000, // 1 digit after the dot
+ 10_000_000, // 2 digits after the dot
+ 1_000_000, // 3 digits after the dot
+ 100_000, // 4 digits after the dot
+ 10_000, // 5 digits after the dot
+ 1000, // 6 digits after the dot
+ 100, // 7 digits after the dot
+ 10, // 8 digits after the dot
+ 1, // 9 digits after the dot
+ };
+
+ private static final AvroDecimalConverter DECIMAL_CONVERTER = new AvroDecimalConverter();
+ private final SynthesizedColumnHandler synthesizedColumnHandler;
+
+ private final List<HiveColumnHandle> columnHandles;
+ private final List<Type> columnTypes;
+ private final Schema schema;
+
+ public HudiAvroSerializer(List<HiveColumnHandle> columnHandles, SynthesizedColumnHandler synthesizedColumnHandler)
+ {
+ this.columnHandles = columnHandles;
+ this.columnTypes = columnHandles.stream().map(HiveColumnHandle::getType).toList();
+ // Fetches projected schema
+ this.schema = constructSchema(columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getName).toList(),
+ columnHandles.stream().filter(ch -> !ch.isHidden()).map(HiveColumnHandle::getHiveType).toList());
+ this.synthesizedColumnHandler = synthesizedColumnHandler;
+ }
+
+ public IndexedRecord serialize(Page sourcePage, int position)
+ {
+ IndexedRecord record = new GenericData.Record(schema);
+ for (int i = 0; i < columnTypes.size(); i++) {
+ Object value = getValue(sourcePage, i, position);
+ record.put(i, value);
+ }
+ return record;
+ }
+
+ public Object getValue(Page sourcePage, int channel, int position)
+ {
+ return columnTypes.get(channel).getObjectValue(null, sourcePage.getBlock(channel), position);
+ }
+
+ public void buildRecordInPage(PageBuilder pageBuilder, IndexedRecord record)
+ {
+ pageBuilder.declarePosition();
+ int blockSeq = 0;
+ for (int channel = 0; channel < columnTypes.size(); channel++, blockSeq++) {
+ BlockBuilder output = pageBuilder.getBlockBuilder(blockSeq);
+ HiveColumnHandle columnHandle = columnHandles.get(channel);
+ if (synthesizedColumnHandler.isSynthesizedColumn(columnHandle)) {
+ synthesizedColumnHandler.getColumnStrategy(columnHandle).appendToBlock(output, columnTypes.get(channel));
+ }
+ else {
+ // Record may not be projected, get index from it
+ int fieldPosInSchema = record.getSchema().getField(columnHandle.getName()).pos();
+ appendTo(columnTypes.get(channel), record.get(fieldPosInSchema), output);
+ }
+ }
+ }
+
+ public static void appendTo(Type type, Object value, BlockBuilder output)
+ {
+ if (value == null) {
+ output.appendNull();
+ return;
+ }
+
+ Class<?> javaType = type.getJavaType();
+ try {
+ if (javaType == boolean.class) {
+ type.writeBoolean(output, (Boolean) value);
+ }
+ else if (javaType == long.class) {
+ if (type.equals(BIGINT)) {
+ type.writeLong(output, ((Number) value).longValue());
+ }
+ else if (type.equals(INTEGER)) {
+ type.writeLong(output, ((Number) value).intValue());
+ }
+ else if (type.equals(SMALLINT)) {
+ type.writeLong(output, ((Number) value).shortValue());
+ }
+ else if (type.equals(TINYINT)) {
+ type.writeLong(output, ((Number) value).byteValue());
+ }
+ else if (type.equals(REAL)) {
+ if (value instanceof Number) {
+ // Directly get the float value from the Number
+ // This preserves the fractional part
+ float floatValue = ((Number) value).floatValue();
+
+ // Get the IEEE 754 single-precision 32-bit representation of this float
+ int intBits = Float.floatToRawIntBits(floatValue);
+
+ // The writeLong method expects these int bits, passed as a long
+ // NOTE: Java handles the widening conversion from int to long
+ type.writeLong(output, intBits);
+ }
+ else {
+ // Handle cases where 'value' is not a Number
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", javaType.getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (type instanceof DecimalType decimalType) {
+ if (value instanceof SqlDecimal sqlDecimal) {
+ if (decimalType.isShort()) {
+ writeShortDecimal(output, sqlDecimal.toBigDecimal().unscaledValue().longValue());
+ }
+ else {
+ writeBigDecimal(decimalType, output, sqlDecimal.toBigDecimal());
+ }
+ }
+ else if (value instanceof GenericData.Fixed fixed) {
+ verify(decimalType.isShort(), "The type should be short decimal");
+ BigDecimal decimal = DECIMAL_CONVERTER.convert(decimalType.getPrecision(), decimalType.getScale(), fixed.bytes());
+ type.writeLong(output, encodeShortScaledValue(decimal, decimalType.getScale()));
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", javaType.getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (type.equals(DATE)) {
+ if (value instanceof SqlDate sqlDate) {
+ type.writeLong(output, sqlDate.getDays());
+ }
+ else if (value instanceof Integer days) {
+ ((DateType) type).writeInt(output, days);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", javaType.getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (type.equals(TIMESTAMP_MICROS)) {
+ type.writeLong(output, toTrinoTimestamp(((Utf8) value).toString()));
+ }
+ else if (type.equals(TIME_MICROS)) {
+ type.writeLong(output, (long) value * PICOSECONDS_PER_MICROSECOND);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Unhandled type for %s: %s", javaType.getSimpleName(), type));
+ }
+ }
+ else if (javaType == double.class) {
+ type.writeDouble(output, ((Number) value).doubleValue());
+ }
+ else if (type.getJavaType() == Int128.class) {
+ writeObject(output, type, value);
+ }
+ else if (javaType == Slice.class) {
+ writeSlice(output, type, value);
+ }
+ else if (javaType == LongTimestamp.class) {
+ if (value instanceof SqlTimestamp sqlTimestamp) {
+ // value is read from parquet
+ // From tests, sqlTimestamp is a UTC epoch that is converted from ZoneId#systemDefault()
+ // IMPORTANT: Even when session's zoneId != ZoneId#systemDefault(), ZoneId#systemDefault() is used calculate/produce the false UTC.
+ // The current sqlTimestamp is calculated as such:
+ // 1. The true UTC timestamp that is stored in file is assumed to be in the local timezone
+ // 2. Trino will them attempt to convert this to a false UTC by subtracting the timezone's offset (factoring offset rules like DST)
+ // Hence, to calculate the true UTC, we will just have to reverse the steps
+
+ // Reconstruct the original local wall time from sqlTimestamp's fields
+ long microsFromSqlTs = sqlTimestamp.getEpochMicros();
+ // picosFromSqlTs is defined as "picoseconds within the microsecond" (0 to 999,999)
+ int picosFromSqlTs = sqlTimestamp.getPicosOfMicros();
+ long secondsComponent = microsFromSqlTs / 1_000_000L;
+ // Storing nanos component separately from seconds component, hence the modulo to remove secondsComponent
+ int nanosComponent = (int) ((microsFromSqlTs % 1_000_000L) * 1000L + picosFromSqlTs / 1000L);
+ LocalDateTime originalLocalWallTime = LocalDateTime.ofEpochSecond(secondsComponent, nanosComponent, ZoneOffset.UTC);
+
+ // Determine the ZoneId in which originalLocalWallTime was observed
+ ZoneId assumedOriginalZoneId = ZoneId.systemDefault();
+
+ // Convert to true UTC by interpreting the local wall time in its original zone
+ // This correctly handles DST for that zone at that specific historical date/time.
+ ZonedDateTime zdtInOriginalZone;
+ try {
+ zdtInOriginalZone = originalLocalWallTime.atZone(assumedOriginalZoneId);
+ }
+ catch (DateTimeException e) {
+ // Handle cases where the local time is invalid in the zone (e.g., during DST "spring forward" gap) or ambiguous (during DST "fall back" overlap).
+ // For now, rethrow or log, as robustly handling this requires a defined policy.
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Cannot uniquely or validly map local time %s to zone %s: %s",
+ originalLocalWallTime, assumedOriginalZoneId, e.getMessage()), e);
+ }
+ Instant trueUtcInstant = zdtInOriginalZone.toInstant();
+
+ // Extract true UTC epoch micros and picos
+ long trueUtcEpochSeconds = trueUtcInstant.getEpochSecond();
+ long trueUtcEpochMicrosContributionFromSeconds;
+ try {
+ trueUtcEpochMicrosContributionFromSeconds = Math.multiplyExact(trueUtcEpochSeconds, 1_000_000L);
+ }
+ catch (ArithmeticException e) {
+ // Multiplication could overflow if epochSeconds is approximately more than 292,271 years from epoch
+ throw new TrinoException(NUMERIC_VALUE_OUT_OF_RANGE,
+ "Timestamp " + trueUtcInstant + " is too far in the past or future to be represented as microseconds in a long.", e);
+ }
+
+ long trueUtcEpochMicrosContributionFromNanos = trueUtcInstant.getNano() / 1000L;
+ long trueUtcEpochMicros;
+
+ try {
+ trueUtcEpochMicros = Math.addExact(trueUtcEpochMicrosContributionFromSeconds, trueUtcEpochMicrosContributionFromNanos);
+ }
+ catch (ArithmeticException e) {
+ // Addition could also theoretically overflow if epochMicrosContributionFromSeconds is:
+ // 1. Very close to Long.MAX_VALUE and trueUtcEpochMicrosContributionFromNanos is positive
+ // 2. Very close to Long.MIN_VALUE and trueUtcEpochMicrosContributionFromNanos is negative
+ throw new TrinoException(NUMERIC_VALUE_OUT_OF_RANGE,
+ "Timestamp " + trueUtcInstant + " results in microsecond representation overflow after adding nanosecond component.", e);
+ }
+
+ int truePicosOfMicros = (trueUtcInstant.getNano() % 1000) * 1000;
+ ((Fixed12BlockBuilder) output).writeFixed12(trueUtcEpochMicros, truePicosOfMicros);
+ }
+ else if (value instanceof Long epochMicros) {
+ // value is read from log
+ // epochMicros is in micros, no nanos or picos component
+ ((Fixed12BlockBuilder) output).writeFixed12(epochMicros, 0);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", javaType.getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (javaType == LongTimestampWithTimeZone.class) {
+ verify(type.equals(TIMESTAMP_TZ_MICROS));
+ long epochMicros = (long) value;
+ int picosOfMillis = toIntExact(floorMod(epochMicros, MICROSECONDS_PER_MILLISECOND)) * PICOSECONDS_PER_MICROSECOND;
+ type.writeObject(output, fromEpochMillisAndFraction(floorDiv(epochMicros, MICROSECONDS_PER_MILLISECOND), picosOfMillis, UTC_KEY));
+ }
+ else if (type instanceof ArrayType arrayType) {
+ writeArray((ArrayBlockBuilder) output, (List<?>) value, arrayType);
+ }
+ else if (type instanceof RowType rowType) {
+ if (value instanceof List<?> list) {
+ // value is read from parquet
+ writeRow((RowBlockBuilder) output, rowType, list);
+ }
+ else if (value instanceof GenericRecord record) {
+ // value is read from log
+ writeRow((RowBlockBuilder) output, rowType, record);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", javaType.getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (type instanceof MapType mapType) {
+ writeMap((MapBlockBuilder) output, mapType, (Map<?, ?>) value);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR, format("Unhandled type for %s: %s", javaType.getSimpleName(), type));
+ }
+ }
+ catch (ClassCastException cce) {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR, format("ClassCastException for type %s: %s with error %s", javaType.getSimpleName(), type, cce));
+ }
+ }
+
+ public static LocalDateTime toLocalDateTime(String datetime)
+ {
+ int dotPosition = datetime.indexOf('.');
+ if (dotPosition == -1) {
+ // no sub-second element
+ return LocalDateTime.from(DateTimeFormatter.ISO_LOCAL_DATE_TIME.parse(datetime));
+ }
+ LocalDateTime result = LocalDateTime.from(DateTimeFormatter.ISO_LOCAL_DATE_TIME.parse(datetime.substring(0, dotPosition)));
+ // has sub-second element, so convert to nanosecond
+ String nanosStr = datetime.substring(dotPosition + 1);
+ int nanoOfSecond = parseInt(nanosStr) * NANO_FACTOR[nanosStr.length()];
+ return result.withNano(nanoOfSecond);
+ }
+
+ public static long toTrinoTimestamp(String datetime)
+ {
+ Instant instant = toLocalDateTime(datetime).toInstant(UTC);
+ return (instant.getEpochSecond() * MICROSECONDS_PER_SECOND) + (instant.getNano() / NANOSECONDS_PER_MICROSECOND);
+ }
+
+ private static void writeSlice(BlockBuilder output, Type type, Object value)
+ {
+ if (type instanceof VarcharType) {
+ if (value instanceof Utf8) {
+ type.writeSlice(output, utf8Slice(((Utf8) value).toString()));
+ }
+ else if (value instanceof String) {
+ type.writeSlice(output, utf8Slice((String) value));
+ }
+ else {
+ type.writeSlice(output, utf8Slice(value.toString()));
+ }
+ }
+ else if (type instanceof VarbinaryType) {
+ if (value instanceof ByteBuffer) {
+ type.writeSlice(output, Slices.wrappedHeapBuffer((ByteBuffer) value));
+ }
+ else if (value instanceof SqlVarbinary sqlVarbinary) {
+ type.writeSlice(output, Slices.wrappedBuffer(sqlVarbinary.getBytes()));
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", type.getJavaType().getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+ else if (type instanceof CharType) {
+ String stringValue;
+ if (value instanceof Utf8) {
+ stringValue = ((Utf8) value).toString();
+ }
+ else if (value instanceof String) {
+ stringValue = (String) value;
+ }
+ else {
+ // Fallback: convert any other object to its string representation
+ stringValue = value.toString();
+ }
+ // IMPORTANT: Char types may be padded with trailing "space" characters to make up for length if the contents are lesser than defined length.
+ // Need to trim out trailing spaces as Slice representing Char should not have trailing spaces
+ type.writeSlice(output, utf8Slice(stringValue.trim()));
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", type.getJavaType().getSimpleName(), type, value.getClass().getName()));
+ }
+ }
+
+ private static void writeObject(BlockBuilder output, Type type, Object value)
+ {
+ if (type instanceof DecimalType decimalType) {
+ BigDecimal valueAsBigDecimal;
+ if (value instanceof SqlDecimal sqlDecimal) {
+ valueAsBigDecimal = sqlDecimal.toBigDecimal();
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR,
+ format("Unhandled type for %s: %s | value type: %s", type.getJavaType().getSimpleName(), type, value.getClass().getName()));
+ }
+
+ Object trinoNativeDecimalValue = Decimals.encodeScaledValue(valueAsBigDecimal, decimalType.getScale());
+ type.writeObject(output, trinoNativeDecimalValue);
+ }
+ else {
+ throw new TrinoException(GENERIC_INTERNAL_ERROR, "Unhandled type for Object: " + type.getTypeSignature());
+ }
+ }
+
+ private static void writeArray(ArrayBlockBuilder output, List<?> value, ArrayType arrayType)
+ {
+ Type elementType = arrayType.getElementType();
+ output.buildEntry(elementBuilder -> {
+ for (Object element : value) {
+ appendTo(elementType, element, elementBuilder);
+ }
+ });
+ }
+
+ private static void writeRow(RowBlockBuilder output, RowType rowType, GenericRecord record)
+ {
+ List<RowType.Field> fields = rowType.getFields();
+ output.buildEntry(fieldBuilders -> {
+ for (int index = 0; index < fields.size(); index++) {
+ RowType.Field field = fields.get(index);
+ appendTo(field.getType(), record.get(field.getName().orElse("field" + index)), fieldBuilders.get(index));
+ }
+ });
+ }
+
+ private static void writeRow(RowBlockBuilder output, RowType rowType, List<?> list)
+ {
+ List<RowType.Field> fields = rowType.getFields();
+ output.buildEntry(fieldBuilders -> {
+ for (int index = 0; index < fields.size(); index++) {
+ RowType.Field field = fields.get(index);
+ appendTo(field.getType(), list.get(index), fieldBuilders.get(index));
+ }
+ });
+ }
+
+ private static void writeMap(MapBlockBuilder output, MapType mapType, Map<?, ?> value)
+ {
+ Type keyType = mapType.getKeyType();
+ Type valueType = mapType.getValueType();
+ output.buildEntry((keyBuilder, valueBuilder) -> {
+ for (Map.Entry<?, ?> entry : value.entrySet()) {
+ appendTo(keyType, entry.getKey(), keyBuilder);
+ appendTo(valueType, entry.getValue(), valueBuilder);
+ }
+ });
+ }
+
+ static class AvroDecimalConverter
+ {
+ private static final Conversions.DecimalConversion AVRO_DECIMAL_CONVERSION = new Conversions.DecimalConversion();
+
+ BigDecimal convert(int precision, int scale, byte[] bytes)
+ {
+ Schema schema = new Schema.Parser().parse(format("{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":%d,\"scale\":%d}", precision, scale));
+ return AVRO_DECIMAL_CONVERSION.fromBytes(ByteBuffer.wrap(bytes), schema, schema.getLogicalType());
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java
new file mode 100644
index 000000000000..c26dd4ca98a8
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/HudiTableTypeUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.util;
+
+import io.trino.plugin.hudi.HudiErrorCode;
+import io.trino.spi.TrinoException;
+import org.apache.hudi.common.model.HoodieTableType;
+
+public class HudiTableTypeUtils
+{
+ private static final String HUDI_PARQUET_INPUT_FORMAT = "org.apache.hudi.hadoop.HoodieParquetInputFormat";
+ private static final String HUDI_PARQUET_REALTIME_INPUT_FORMAT = "org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat";
+ private static final String HUDI_INPUT_FORMAT = "com.uber.hoodie.hadoop.HoodieInputFormat";
+ private static final String HUDI_REALTIME_INPUT_FORMAT = "com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat";
+
+ private HudiTableTypeUtils()
+ {
+ }
+
+ public static HoodieTableType fromInputFormat(String inputFormat)
+ {
+ switch (inputFormat) {
+ case HUDI_PARQUET_INPUT_FORMAT:
+ case HUDI_INPUT_FORMAT:
+ return HoodieTableType.COPY_ON_WRITE;
+ case HUDI_PARQUET_REALTIME_INPUT_FORMAT:
+ case HUDI_REALTIME_INPUT_FORMAT:
+ return HoodieTableType.MERGE_ON_READ;
+ default:
+ throw new TrinoException(HudiErrorCode.HUDI_UNSUPPORTED_TABLE_TYPE, "Table has an unsupported input format: " + inputFormat);
+ }
+ }
+}
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java
new file mode 100644
index 000000000000..5d55adb5577c
--- /dev/null
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/util/SynthesizedColumnHandler.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.util;
+
+import com.google.common.collect.ImmutableMap;
+import io.trino.plugin.hive.HiveColumnHandle;
+import io.trino.plugin.hive.HivePartitionKey;
+import io.trino.plugin.hudi.HudiSplit;
+import io.trino.plugin.hudi.file.HudiFile;
+import io.trino.spi.TrinoException;
+import io.trino.spi.block.Block;
+import io.trino.spi.block.BlockBuilder;
+import io.trino.spi.block.RunLengthEncodedBlock;
+import io.trino.spi.type.BigintType;
+import io.trino.spi.type.BooleanType;
+import io.trino.spi.type.DateType;
+import io.trino.spi.type.DecimalType;
+import io.trino.spi.type.IntegerType;
+import io.trino.spi.type.SqlDecimal;
+import io.trino.spi.type.TimestampWithTimeZoneType;
+import io.trino.spi.type.Type;
+import io.trino.spi.type.VarcharType;
+
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.format.DateTimeParseException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static io.airlift.slice.Slices.utf8Slice;
+import static io.trino.metastore.Partitions.makePartName;
+import static io.trino.plugin.hive.HiveColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME;
+import static io.trino.plugin.hive.HiveColumnHandle.FILE_SIZE_COLUMN_NAME;
+import static io.trino.plugin.hive.HiveColumnHandle.PARTITION_COLUMN_NAME;
+import static io.trino.plugin.hive.HiveColumnHandle.PATH_COLUMN_NAME;
+import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
+import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone;
+import static io.trino.spi.type.Decimals.writeBigDecimal;
+import static io.trino.spi.type.Decimals.writeShortDecimal;
+import static io.trino.spi.type.TimeZoneKey.UTC_KEY;
+import static java.lang.Math.toIntExact;
+import static java.lang.String.format;
+
+/**
+ * Handles synthesized (virtual) columns in Hudi tables, such as partition columns and metadata (not hudi metadata)
+ * columns.
+ */
+public class SynthesizedColumnHandler
+{
+ private final Map<String, SynthesizedColumnStrategy> strategies;
+ private final SplitMetadata splitMetadata;
+
+ public static SynthesizedColumnHandler create(HudiSplit hudiSplit)
+ {
+ return new SynthesizedColumnHandler(hudiSplit);
+ }
+
+ /**
+ * Constructs a SynthesizedColumnHandler with the given partition keys.
+ */
+ public SynthesizedColumnHandler(HudiSplit hudiSplit)
+ {
+ this.splitMetadata = SplitMetadata.of(hudiSplit);
+ ImmutableMap.Builder<String, SynthesizedColumnStrategy> builder = ImmutableMap.builder();
+ initSynthesizedColStrategies(builder);
+ initPartitionKeyStrategies(builder, hudiSplit);
+ strategies = builder.buildOrThrow();
+ }
+
+ /**
+ * Initializes strategies for synthesized columns.
+ */
+ private void initSynthesizedColStrategies(ImmutableMap.Builder<String, SynthesizedColumnStrategy> builder)
+ {
... 6906 lines suppressed ...