You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2025/08/27 11:48:51 UTC

(hudi) 13/13: [ENG-30507] Incorrect query results for Merge-On-Read (RT) tables when column stats are enabled (#58)

This is an automated email from the ASF dual-hosted git repository.

vhs pushed a commit to branch HUDI-9767__upstream-trino-changes
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 194e58526b6992b878c13e72fbcf2fcbd0bcc9da
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Mon Aug 25 22:25:43 2025 +0800

    [ENG-30507] Incorrect query results for Merge-On-Read (RT) tables when column stats are enabled (#58)
    
    * Co-authored-by: Vamsi <va...@onehouse.ai>
    
    (cherry picked from commit 5e3ebd5b4edd0623e041109cc769f0123bcbd4a7)
---
 .../query/index/HudiColumnStatsIndexSupport.java   |  23 ++++--
 .../io/trino/plugin/hudi/TestHudiSmokeTest.java    |  30 ++++++++
 .../testing/ResourceHudiTablesInitializer.java     |   9 +++
 .../hudi-testing-data/hudi_non_part_mor.md         |  77 +++++++++++++++++++++
 .../hudi-testing-data/hudi_non_part_mor.zip        | Bin 0 -> 82188 bytes
 5 files changed, 133 insertions(+), 6 deletions(-)

diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
index 9b3291b6fc4b..0d1c9eaa6cd0 100644
--- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
@@ -203,13 +203,24 @@ public class HudiColumnStatsIndexSupport
             TupleDomain<String> regularColumnPredicates,
             List<String> regularColumns)
     {
-        String fileSliceName = fileSlice.getBaseFile().map(BaseFile::getFileName).orElse("");
-        // If no stats exist for this specific file, we cannot prune it.
-        if (!domainsWithStats.containsKey(fileSliceName)) {
-            return false;
+        List<String> filesToLookUp = new ArrayList<>();
+        fileSlice.getBaseFile()
+                .map(BaseFile::getFileName)
+                .ifPresent(filesToLookUp::add);
+
+        if (fileSlice.hasLogFiles()) {
+            fileSlice.getLogFiles().forEach(logFile -> filesToLookUp.add(logFile.getFileName()));
         }
-        Map<String, Domain> fileDomainsWithStats = domainsWithStats.get(fileSliceName);
-        return !evaluateStatisticPredicate(regularColumnPredicates, fileDomainsWithStats, regularColumns);
+
+        // if any log or base file in the file slice matches the predicate, all files in the file slice needs to be read
+        return filesToLookUp.stream().allMatch(fileName -> {
+            // If no stats exist for this specific file, we cannot prune it.
+            if (!domainsWithStats.containsKey(fileName)) {
+                return false;
+            }
+            Map<String, Domain> fileDomainsWithStats = domainsWithStats.get(fileName);
+            return !evaluateStatisticPredicate(regularColumnPredicates, fileDomainsWithStats, regularColumns);
+        });
     }
 
     protected static boolean evaluateStatisticPredicate(
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
index 3c98a85f5b2a..1080d5e89b7b 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
@@ -80,6 +80,7 @@ import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.Testing
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_PT_V8_MOR;
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_EXTRACTABLE_PARTITION_PATH;
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_PART_COW;
+import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_PART_MOR;
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_STOCK_TICKS_COW;
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_STOCK_TICKS_MOR;
 import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_TIMESTAMP_KEYGEN_PT_EPOCH_TO_YYYY_MM_DD_HH_V8_MOR;
@@ -684,6 +685,35 @@ public class TestHudiSmokeTest
         assertThat(prunedSplits).isEqualTo(1);
     }
 
+    @Test
+    public void testColStatsFileSkippingMORTable()
+    {
+        Session session = SessionBuilder.from(getSession())
+                .withMdtEnabled(true)
+                .withColStatsIndexEnabled(true)
+                .withColumnStatsTimeout("1s")
+                .withRecordLevelIndexEnabled(false)
+                .withSecondaryIndexEnabled(false)
+                .withPartitionStatsIndexEnabled(false)
+                .build();
+        MaterializedResult roTableRes = getQueryRunner().execute(session, "SELECT id, name FROM " + HUDI_NON_PART_MOR + " where name = 'Alice'");
+        MaterializedResult rtTableRes = getQueryRunner().execute(session, "SELECT id, name FROM " + HUDI_NON_PART_MOR + "_rt where name = 'Cathy'");
+
+        // verify ro table returns results from base file
+        int roTableSplits = roTableRes.getStatementStats().get().getTotalSplits();
+        int roTableRows = roTableRes.getRowCount();
+        assertThat(roTableSplits).isEqualTo(1);
+        assertThat(roTableRows).isEqualTo(1);
+        assertThat(roTableRes.getMaterializedRows().getFirst().getField(1)).isEqualTo("Alice");
+
+        // verify rt table returns results from log file
+        int rtTableSplits = rtTableRes.getStatementStats().get().getTotalSplits();
+        int rtTableRows = rtTableRes.getRowCount();
+        assertThat(rtTableRows).isEqualTo(1);
+        assertThat(rtTableSplits).isEqualTo(1);
+        assertThat(rtTableRes.getMaterializedRows().getFirst().getField(1)).isEqualTo("Cathy");
+    }
+
     @ParameterizedTest
     @EnumSource(
             value = ResourceHudiTablesInitializer.TestingTable.class,
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
index 420e10e1ada6..cebbd7a51d2a 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
@@ -333,6 +333,7 @@ public class ResourceHudiTablesInitializer
     public enum TestingTable
     {
         HUDI_NON_PART_COW(nonPartitionRegularColumns()),
+        HUDI_NON_PART_MOR(simpleRegularColumns(), ImmutableList.of(), ImmutableMap.of(), true),
         HUDI_TRIPS_COW_V8(tripsRegularColumns()),
         HUDI_COW_TABLE_WITH_FIELD_NAMES_IN_CAPS(hudiTableWithFieldNamesInCapsRegularColumns()),
         HUDI_COW_PT_TABLE_WITH_FIELD_NAMES_IN_CAPS(hudiTableWithFieldNamesInCapsRegularColumns(), hudiTableWithFieldNamesInCapsPartitionColumns(), hudiTableWithFieldNamesInCapsPartitions(), false), // delete
@@ -468,6 +469,14 @@ public class ResourceHudiTablesInitializer
             return ImmutableList.of(column("country", HIVE_STRING));
         }
 
+        private static List<Column> simpleRegularColumns()
+        {
+            return ImmutableList.of(
+                    column("id", HIVE_STRING),
+                    column("name", HIVE_STRING),
+                    column("age", HIVE_INT));
+        }
+
         private static List<Column> tripsRegularColumns()
         {
             return ImmutableList.of(
diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md
new file mode 100644
index 000000000000..90b168a74156
--- /dev/null
+++ b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md
@@ -0,0 +1,77 @@
+# Hudi Test Resources
+
+## Generating Hudi Resources
+
+Follow these steps to create the `hudi_non_part_mor` test table and utilize it for testing.
+
+### Download and install spark version 3.3 - 3.5
+### Open spark scala shell
+
+Execute the following command in the terminal to create the table locally:
+
+### Generate Resources
+
+* Open the `spark-shell` terminal using below command
+```
+./spark-shell \
+  --packages org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.2,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.538 \
+  --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \
+  --conf 'spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog' \
+  --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' \
+  --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar'
+```
+* Execute the following Spark scala code to create the `hudi_non_part_mor` table:
+
+```
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.Row
+
+val schema = StructType(Seq(
+  StructField("id", StringType, nullable = false),
+  StructField("name", StringType, nullable = true),
+  StructField("age", IntegerType, nullable = true)
+))
+
+
+val data = Seq(
+  Row("1", "Alice", 30),
+  Row("2", "Bob", 25)
+)
+
+val df = spark.createDataFrame(
+  spark.sparkContext.parallelize(data),
+  schema
+)
+
+var basePath = "file:///tmp/hudi_non_part_mor"
+
+df.write()
+  .format("hudi")
+  .mode("Append")
+  .option("hoodie.table.name", "hudi_non_part_mor")
+  .option("hoodie.datasource.write.table.type", "MERGE_ON_READ")
+  .option("hoodie.datasource.write.recordkey.field", "id")
+  .option("hoodie.datasource.write.operation", "bulk_insert")
+  .option("hoodie.metadata.index.column.stats.enable", "true")
+  .save(basePath);
+
+val data = Seq(
+  Row("1", "Cathy", 30),
+  Row("2", "David", 25)
+)
+
+val df = spark.createDataFrame(
+  spark.sparkContext.parallelize(data),
+  schema
+)
+
+df.write
+    .format("hudi")
+    .mode("Append")
+    .option("hoodie.table.name", "hudi_non_part_mor")
+    .option("hoodie.datasource.write.table.type", "MERGE_ON_READ")
+    .option("hoodie.datasource.write.recordkey.field","id")
+    .option("hoodie.datasource.write.operation","upsert")
+    .option("hoodie.metadata.index.column.stats.enable", "true")
+    .save(basePath)
+```
diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip
new file mode 100644
index 000000000000..521f1583beb4
Binary files /dev/null and b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip differ