You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2025/08/27 11:48:51 UTC
(hudi) 13/13: [ENG-30507] Incorrect query results for Merge-On-Read (RT) tables when column stats are enabled (#58)
This is an automated email from the ASF dual-hosted git repository.
vhs pushed a commit to branch HUDI-9767__upstream-trino-changes
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 194e58526b6992b878c13e72fbcf2fcbd0bcc9da
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Mon Aug 25 22:25:43 2025 +0800
[ENG-30507] Incorrect query results for Merge-On-Read (RT) tables when column stats are enabled (#58)
* Co-authored-by: Vamsi <va...@onehouse.ai>
(cherry picked from commit 5e3ebd5b4edd0623e041109cc769f0123bcbd4a7)
---
.../query/index/HudiColumnStatsIndexSupport.java | 23 ++++--
.../io/trino/plugin/hudi/TestHudiSmokeTest.java | 30 ++++++++
.../testing/ResourceHudiTablesInitializer.java | 9 +++
.../hudi-testing-data/hudi_non_part_mor.md | 77 +++++++++++++++++++++
.../hudi-testing-data/hudi_non_part_mor.zip | Bin 0 -> 82188 bytes
5 files changed, 133 insertions(+), 6 deletions(-)
diff --git a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
index 9b3291b6fc4b..0d1c9eaa6cd0 100644
--- a/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
+++ b/hudi-trino-plugin/src/main/java/io/trino/plugin/hudi/query/index/HudiColumnStatsIndexSupport.java
@@ -203,13 +203,24 @@ public class HudiColumnStatsIndexSupport
TupleDomain<String> regularColumnPredicates,
List<String> regularColumns)
{
- String fileSliceName = fileSlice.getBaseFile().map(BaseFile::getFileName).orElse("");
- // If no stats exist for this specific file, we cannot prune it.
- if (!domainsWithStats.containsKey(fileSliceName)) {
- return false;
+ List<String> filesToLookUp = new ArrayList<>();
+ fileSlice.getBaseFile()
+ .map(BaseFile::getFileName)
+ .ifPresent(filesToLookUp::add);
+
+ if (fileSlice.hasLogFiles()) {
+ fileSlice.getLogFiles().forEach(logFile -> filesToLookUp.add(logFile.getFileName()));
}
- Map<String, Domain> fileDomainsWithStats = domainsWithStats.get(fileSliceName);
- return !evaluateStatisticPredicate(regularColumnPredicates, fileDomainsWithStats, regularColumns);
+
+ // if any log or base file in the file slice matches the predicate, all files in the file slice needs to be read
+ return filesToLookUp.stream().allMatch(fileName -> {
+ // If no stats exist for this specific file, we cannot prune it.
+ if (!domainsWithStats.containsKey(fileName)) {
+ return false;
+ }
+ Map<String, Domain> fileDomainsWithStats = domainsWithStats.get(fileName);
+ return !evaluateStatisticPredicate(regularColumnPredicates, fileDomainsWithStats, regularColumns);
+ });
}
protected static boolean evaluateStatisticPredicate(
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
index 3c98a85f5b2a..1080d5e89b7b 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiSmokeTest.java
@@ -80,6 +80,7 @@ import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.Testing
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_PT_V8_MOR;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_EXTRACTABLE_PARTITION_PATH;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_PART_COW;
+import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_NON_PART_MOR;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_STOCK_TICKS_COW;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_STOCK_TICKS_MOR;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_TIMESTAMP_KEYGEN_PT_EPOCH_TO_YYYY_MM_DD_HH_V8_MOR;
@@ -684,6 +685,35 @@ public class TestHudiSmokeTest
assertThat(prunedSplits).isEqualTo(1);
}
+ @Test
+ public void testColStatsFileSkippingMORTable()
+ {
+ Session session = SessionBuilder.from(getSession())
+ .withMdtEnabled(true)
+ .withColStatsIndexEnabled(true)
+ .withColumnStatsTimeout("1s")
+ .withRecordLevelIndexEnabled(false)
+ .withSecondaryIndexEnabled(false)
+ .withPartitionStatsIndexEnabled(false)
+ .build();
+ MaterializedResult roTableRes = getQueryRunner().execute(session, "SELECT id, name FROM " + HUDI_NON_PART_MOR + " where name = 'Alice'");
+ MaterializedResult rtTableRes = getQueryRunner().execute(session, "SELECT id, name FROM " + HUDI_NON_PART_MOR + "_rt where name = 'Cathy'");
+
+ // verify ro table returns results from base file
+ int roTableSplits = roTableRes.getStatementStats().get().getTotalSplits();
+ int roTableRows = roTableRes.getRowCount();
+ assertThat(roTableSplits).isEqualTo(1);
+ assertThat(roTableRows).isEqualTo(1);
+ assertThat(roTableRes.getMaterializedRows().getFirst().getField(1)).isEqualTo("Alice");
+
+ // verify rt table returns results from log file
+ int rtTableSplits = rtTableRes.getStatementStats().get().getTotalSplits();
+ int rtTableRows = rtTableRes.getRowCount();
+ assertThat(rtTableRows).isEqualTo(1);
+ assertThat(rtTableSplits).isEqualTo(1);
+ assertThat(rtTableRes.getMaterializedRows().getFirst().getField(1)).isEqualTo("Cathy");
+ }
+
@ParameterizedTest
@EnumSource(
value = ResourceHudiTablesInitializer.TestingTable.class,
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
index 420e10e1ada6..cebbd7a51d2a 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/ResourceHudiTablesInitializer.java
@@ -333,6 +333,7 @@ public class ResourceHudiTablesInitializer
public enum TestingTable
{
HUDI_NON_PART_COW(nonPartitionRegularColumns()),
+ HUDI_NON_PART_MOR(simpleRegularColumns(), ImmutableList.of(), ImmutableMap.of(), true),
HUDI_TRIPS_COW_V8(tripsRegularColumns()),
HUDI_COW_TABLE_WITH_FIELD_NAMES_IN_CAPS(hudiTableWithFieldNamesInCapsRegularColumns()),
HUDI_COW_PT_TABLE_WITH_FIELD_NAMES_IN_CAPS(hudiTableWithFieldNamesInCapsRegularColumns(), hudiTableWithFieldNamesInCapsPartitionColumns(), hudiTableWithFieldNamesInCapsPartitions(), false), // delete
@@ -468,6 +469,14 @@ public class ResourceHudiTablesInitializer
return ImmutableList.of(column("country", HIVE_STRING));
}
+ private static List<Column> simpleRegularColumns()
+ {
+ return ImmutableList.of(
+ column("id", HIVE_STRING),
+ column("name", HIVE_STRING),
+ column("age", HIVE_INT));
+ }
+
private static List<Column> tripsRegularColumns()
{
return ImmutableList.of(
diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md
new file mode 100644
index 000000000000..90b168a74156
--- /dev/null
+++ b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.md
@@ -0,0 +1,77 @@
+# Hudi Test Resources
+
+## Generating Hudi Resources
+
+Follow these steps to create the `hudi_non_part_mor` test table and utilize it for testing.
+
+### Download and install spark version 3.3 - 3.5
+### Open spark scala shell
+
+Execute the following command in the terminal to create the table locally:
+
+### Generate Resources
+
+* Open the `spark-shell` terminal using below command
+```
+./spark-shell \
+ --packages org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.2,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.538 \
+ --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \
+ --conf 'spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog' \
+ --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' \
+ --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar'
+```
+* Execute the following Spark scala code to create the `hudi_non_part_mor` table:
+
+```
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.Row
+
+val schema = StructType(Seq(
+ StructField("id", StringType, nullable = false),
+ StructField("name", StringType, nullable = true),
+ StructField("age", IntegerType, nullable = true)
+))
+
+
+val data = Seq(
+ Row("1", "Alice", 30),
+ Row("2", "Bob", 25)
+)
+
+val df = spark.createDataFrame(
+ spark.sparkContext.parallelize(data),
+ schema
+)
+
+var basePath = "file:///tmp/hudi_non_part_mor"
+
+df.write()
+ .format("hudi")
+ .mode("Append")
+ .option("hoodie.table.name", "hudi_non_part_mor")
+ .option("hoodie.datasource.write.table.type", "MERGE_ON_READ")
+ .option("hoodie.datasource.write.recordkey.field", "id")
+ .option("hoodie.datasource.write.operation", "bulk_insert")
+ .option("hoodie.metadata.index.column.stats.enable", "true")
+ .save(basePath);
+
+val data = Seq(
+ Row("1", "Cathy", 30),
+ Row("2", "David", 25)
+)
+
+val df = spark.createDataFrame(
+ spark.sparkContext.parallelize(data),
+ schema
+)
+
+df.write
+ .format("hudi")
+ .mode("Append")
+ .option("hoodie.table.name", "hudi_non_part_mor")
+ .option("hoodie.datasource.write.table.type", "MERGE_ON_READ")
+ .option("hoodie.datasource.write.recordkey.field","id")
+ .option("hoodie.datasource.write.operation","upsert")
+ .option("hoodie.metadata.index.column.stats.enable", "true")
+ .save(basePath)
+```
diff --git a/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip
new file mode 100644
index 000000000000..521f1583beb4
Binary files /dev/null and b/hudi-trino-plugin/src/test/resources/hudi-testing-data/hudi_non_part_mor.zip differ