You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by sl...@apache.org on 2026/05/14 01:16:49 UTC
(auron) branch master updated: [AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 0ca9d958 [AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)
0ca9d958 is described below
commit 0ca9d95834b00691bab34a86622c44c4a7b27538
Author: Ming Wei <we...@gmail.com>
AuthorDate: Thu May 14 09:16:44 2026 +0800
[AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)
# Which issue does this PR close?
Closes #2248
# Rationale for this change
Hudi MOR read-optimized queries only read base files and do not require
log-file merging. Auron can therefore reuse the existing native
Parquet/ORC scan path for this query mode.
# What changes are included in this PR?
- Detect Hudi query type from `hoodie.datasource.query.type`.
- Also support the compatible `hoodie.datasource.view.type` option.
- Allow MOR native scan only for explicit `read_optimized` queries.
- Keep `snapshot`, `incremental`, `realtime`, and time travel queries on
the Spark fallback path.
- Add UT coverage for MOR read-optimized support detection and native
scan execution plan.
# Are there any user-facing changes?
Yes. MOR read-optimized Hudi queries can now use native Parquet/ORC scan
when supported.
# How was this patch tested?
- Added unit test coverage for MOR read-optimized support detection.
- Added correctness and native scan plan validation for a MOR
read-optimized Hudi table scan.
Signed-off-by: weimingdiit <we...@gmail.com>
---
.../spark/sql/auron/hudi/HudiScanSupport.scala | 21 ++++++-
.../sql/auron/hudi/HudiScanSupportSuite.scala | 67 +++++++++++++++++++++-
2 files changed, 85 insertions(+), 3 deletions(-)
diff --git a/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala b/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
index 5074f210..0ebcb142 100644
--- a/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
+++ b/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
@@ -39,10 +39,13 @@ object HudiScanSupport extends Logging {
private val hudiOrcFileFormatSuffix = "HoodieOrcFileFormat"
private val newHudiOrcFileFormatSuffix = "NewHoodieOrcFileFormat"
private val morTableTypes = Set("merge_on_read", "mor")
+ private val readOptimizedQueryTypes = Set("read_optimized")
private val hudiTableTypeKeys = Seq(
"hoodie.datasource.write.table.type",
"hoodie.datasource.read.table.type",
"hoodie.table.type")
+ private val hudiQueryTypeKeys =
+ Seq("hoodie.datasource.query.type", "hoodie.datasource.view.type")
private val hudiBaseFileFormatKeys = Seq(
"hoodie.table.base.file.format",
"hoodie.datasource.write.base.file.format",
@@ -117,17 +120,31 @@ object HudiScanSupport extends Logging {
.orElse(tableTypeFromCatalog(catalogTable))
.orElse(tableTypeFromMeta(tableProperties))
.map(_.toLowerCase(Locale.ROOT))
+ val queryType = queryTypeFromOptions(options).map(_.toLowerCase(Locale.ROOT))
logDebug(s"Hudi tableType resolved to: ${tableType.getOrElse("unknown")}")
- // Only support basic COW tables for the base version.
- !tableType.exists(morTableTypes.contains)
+ val isMor = tableType.exists(morTableTypes.contains)
+ queryType match {
+ case Some(query) if readOptimizedQueryTypes.contains(query) => true
+ case Some("snapshot") => !isMor
+ case Some("incremental" | "realtime") => false
+ case Some(_) => false
+ case None =>
+ // MOR snapshot reads may need log-file merging. Native scan is safe only
+ // when the query explicitly requests read-optimized base-file reads.
+ !isMor
+ }
}
private def tableTypeFromOptions(options: Map[String, String]): Option[String] = {
caseInsensitiveValue(options, hudiTableTypeKeys)
}
+ private def queryTypeFromOptions(options: Map[String, String]): Option[String] = {
+ caseInsensitiveValue(options, hudiQueryTypeKeys)
+ }
+
private[hudi] def baseFileFormatFromOptions(options: Map[String, String]): Option[String] = {
caseInsensitiveValue(options, hudiBaseFileFormatKeys)
}
diff --git a/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala b/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
index 9e660deb..a2721e1d 100644
--- a/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
+++ b/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
@@ -186,6 +186,10 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
Option(props.getProperty("hoodie.table.base.file.format"))
}
+ private def hudiTablePath(tableName: String): String = {
+ new File(warehouseDir, tableName).getAbsolutePath
+ }
+
private def assumeSparkAtLeast(version: String): Unit = {
val current = SparkVersionUtil.SPARK_RUNTIME_VERSION
assume(current >= version, s"Requires Spark >= $version, current Spark $current")
@@ -218,23 +222,48 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
.isEmpty)
}
- test("hudi isSupported rejects MOR table types") {
+ test("hudi isSupported rejects MOR table types unless read optimized") {
val options = Map("hoodie.datasource.write.table.type" -> "MERGE_ON_READ")
assert(
!HudiScanSupport.isSupported(
"org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
options))
+ assert(
+ HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ options + ("hoodie.datasource.query.type" -> "read_optimized")))
+ assert(
+ HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ options + ("Hoodie.DataSource.View.Type" -> "READ_OPTIMIZED")))
+ assert(
+ !HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ options + ("hoodie.datasource.query.type" -> "snapshot")))
+ assert(
+ !HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ options + ("Hoodie.DataSource.View.Type" -> "REALTIME")))
+ assert(
+ !HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ options + ("hoodie.datasource.query.type" -> "incremental")))
}
test("hudi scan options are case-insensitive") {
val options = Map(
"Hoodie.DataSource.Write.Table.Type" -> "MERGE_ON_READ",
"Hoodie.Table.Base.File.Format" -> "ORC")
+ val readOptimizedOptions = options + ("Hoodie.DataSource.Query.Type" -> "READ_OPTIMIZED")
val timeTravelOptions = Map("Hoodie.DataSource.Read.As.Of.Instant" -> "20240101010101")
assert(
!HudiScanSupport.isSupported(
"org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
options))
+ assert(
+ HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ readOptimizedOptions))
assert(
!HudiScanSupport.isSupported(
"org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
@@ -247,6 +276,12 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
HudiScanSupport.isSupported(
"org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
Map.empty))
+ assert(
+ HudiScanSupport.isSupported(
+ "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+ Map(
+ "hoodie.datasource.write.table.type" -> "COPY_ON_WRITE",
+ "hoodie.datasource.query.type" -> "snapshot")))
}
test("hudi isSupported rejects non-Hudi formats") {
@@ -350,4 +385,34 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
assertHasNativeParquetScan(converted)
}
}
+
+ test("hudi: MOR read-optimized table scan converts to native") {
+ withTable("hudi_mor_read_optimized") {
+ spark.sql("""create table hudi_mor_read_optimized (id int, name string)
+ |using hudi
+ |tblproperties (
+ | 'hoodie.datasource.write.table.type' = 'MERGE_ON_READ'
+ |)""".stripMargin)
+ spark.sql("insert into hudi_mor_read_optimized values (1, 'v1'), (2, 'v2')")
+
+ val df = spark.read
+ .format("hudi")
+ .option("hoodie.datasource.query.type", "read_optimized")
+ .load(hudiTablePath("hudi_mor_read_optimized"))
+ .select("id", "name")
+ .orderBy("id")
+
+ logFileFormats(df)
+ val rows = df.collect().toSeq
+ assert(rows == Seq(Row(1, "v1"), Row(2, "v2")))
+ val scan = df.queryExecution.sparkPlan.collectFirst {
+ case s: org.apache.spark.sql.execution.FileSourceScanExec => s
+ }
+ assert(scan.isDefined)
+ val provider = new HudiConvertProvider
+ assert(provider.isSupported(scan.get))
+ val converted = provider.convert(scan.get)
+ assertHasNativeParquetScan(converted)
+ }
+ }
}