You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by sl...@apache.org on 2026/05/14 01:16:49 UTC

(auron) branch master updated: [AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)

This is an automated email from the ASF dual-hosted git repository.

slfan1989 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git


The following commit(s) were added to refs/heads/master by this push:
     new 0ca9d958 [AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)
0ca9d958 is described below

commit 0ca9d95834b00691bab34a86622c44c4a7b27538
Author: Ming Wei <we...@gmail.com>
AuthorDate: Thu May 14 09:16:44 2026 +0800

    [AURON #2248] Support native Hudi scan for MOR read-optimized queries (#2249)
    
    # Which issue does this PR close?
    
    Closes #2248
    
    # Rationale for this change
    
    Hudi MOR read-optimized queries only read base files and do not require
    log-file merging. Auron can therefore reuse the existing native
    Parquet/ORC scan path for this query mode.
    
    # What changes are included in this PR?
    
    - Detect Hudi query type from `hoodie.datasource.query.type`.
    - Also support the compatible `hoodie.datasource.view.type` option.
    - Allow MOR native scan only for explicit `read_optimized` queries.
    - Keep `snapshot`, `incremental`, `realtime`, and time travel queries on
    the Spark fallback path.
    - Add UT coverage for MOR read-optimized support detection and native
    scan execution plan.
    
    # Are there any user-facing changes?
    Yes. MOR read-optimized Hudi queries can now use native Parquet/ORC scan
    when supported.
    
    # How was this patch tested?
    - Added unit test coverage for MOR read-optimized support detection.
    - Added correctness and native scan plan validation for a MOR
    read-optimized Hudi table scan.
    
    Signed-off-by: weimingdiit <we...@gmail.com>
---
 .../spark/sql/auron/hudi/HudiScanSupport.scala     | 21 ++++++-
 .../sql/auron/hudi/HudiScanSupportSuite.scala      | 67 +++++++++++++++++++++-
 2 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala b/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
index 5074f210..0ebcb142 100644
--- a/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
+++ b/thirdparty/auron-hudi/src/main/scala/org/apache/spark/sql/auron/hudi/HudiScanSupport.scala
@@ -39,10 +39,13 @@ object HudiScanSupport extends Logging {
   private val hudiOrcFileFormatSuffix = "HoodieOrcFileFormat"
   private val newHudiOrcFileFormatSuffix = "NewHoodieOrcFileFormat"
   private val morTableTypes = Set("merge_on_read", "mor")
+  private val readOptimizedQueryTypes = Set("read_optimized")
   private val hudiTableTypeKeys = Seq(
     "hoodie.datasource.write.table.type",
     "hoodie.datasource.read.table.type",
     "hoodie.table.type")
+  private val hudiQueryTypeKeys =
+    Seq("hoodie.datasource.query.type", "hoodie.datasource.view.type")
   private val hudiBaseFileFormatKeys = Seq(
     "hoodie.table.base.file.format",
     "hoodie.datasource.write.base.file.format",
@@ -117,17 +120,31 @@ object HudiScanSupport extends Logging {
       .orElse(tableTypeFromCatalog(catalogTable))
       .orElse(tableTypeFromMeta(tableProperties))
       .map(_.toLowerCase(Locale.ROOT))
+    val queryType = queryTypeFromOptions(options).map(_.toLowerCase(Locale.ROOT))
 
     logDebug(s"Hudi tableType resolved to: ${tableType.getOrElse("unknown")}")
 
-    // Only support basic COW tables for the base version.
-    !tableType.exists(morTableTypes.contains)
+    val isMor = tableType.exists(morTableTypes.contains)
+    queryType match {
+      case Some(query) if readOptimizedQueryTypes.contains(query) => true
+      case Some("snapshot") => !isMor
+      case Some("incremental" | "realtime") => false
+      case Some(_) => false
+      case None =>
+        // MOR snapshot reads may need log-file merging. Native scan is safe only
+        // when the query explicitly requests read-optimized base-file reads.
+        !isMor
+    }
   }
 
   private def tableTypeFromOptions(options: Map[String, String]): Option[String] = {
     caseInsensitiveValue(options, hudiTableTypeKeys)
   }
 
+  private def queryTypeFromOptions(options: Map[String, String]): Option[String] = {
+    caseInsensitiveValue(options, hudiQueryTypeKeys)
+  }
+
   private[hudi] def baseFileFormatFromOptions(options: Map[String, String]): Option[String] = {
     caseInsensitiveValue(options, hudiBaseFileFormatKeys)
   }
diff --git a/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala b/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
index 9e660deb..a2721e1d 100644
--- a/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
+++ b/thirdparty/auron-hudi/src/test/scala/org/apache/spark/sql/auron/hudi/HudiScanSupportSuite.scala
@@ -186,6 +186,10 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
     Option(props.getProperty("hoodie.table.base.file.format"))
   }
 
+  private def hudiTablePath(tableName: String): String = {
+    new File(warehouseDir, tableName).getAbsolutePath
+  }
+
   private def assumeSparkAtLeast(version: String): Unit = {
     val current = SparkVersionUtil.SPARK_RUNTIME_VERSION
     assume(current >= version, s"Requires Spark >= $version, current Spark $current")
@@ -218,23 +222,48 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
         .isEmpty)
   }
 
-  test("hudi isSupported rejects MOR table types") {
+  test("hudi isSupported rejects MOR table types unless read optimized") {
     val options = Map("hoodie.datasource.write.table.type" -> "MERGE_ON_READ")
     assert(
       !HudiScanSupport.isSupported(
         "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
         options))
+    assert(
+      HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        options + ("hoodie.datasource.query.type" -> "read_optimized")))
+    assert(
+      HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        options + ("Hoodie.DataSource.View.Type" -> "READ_OPTIMIZED")))
+    assert(
+      !HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        options + ("hoodie.datasource.query.type" -> "snapshot")))
+    assert(
+      !HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        options + ("Hoodie.DataSource.View.Type" -> "REALTIME")))
+    assert(
+      !HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        options + ("hoodie.datasource.query.type" -> "incremental")))
   }
 
   test("hudi scan options are case-insensitive") {
     val options = Map(
       "Hoodie.DataSource.Write.Table.Type" -> "MERGE_ON_READ",
       "Hoodie.Table.Base.File.Format" -> "ORC")
+    val readOptimizedOptions = options + ("Hoodie.DataSource.Query.Type" -> "READ_OPTIMIZED")
     val timeTravelOptions = Map("Hoodie.DataSource.Read.As.Of.Instant" -> "20240101010101")
     assert(
       !HudiScanSupport.isSupported(
         "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
         options))
+    assert(
+      HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        readOptimizedOptions))
     assert(
       !HudiScanSupport.isSupported(
         "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
@@ -247,6 +276,12 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
       HudiScanSupport.isSupported(
         "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
         Map.empty))
+    assert(
+      HudiScanSupport.isSupported(
+        "org.apache.spark.sql.execution.datasources.parquet.HoodieParquetFileFormat",
+        Map(
+          "hoodie.datasource.write.table.type" -> "COPY_ON_WRITE",
+          "hoodie.datasource.query.type" -> "snapshot")))
   }
 
   test("hudi isSupported rejects non-Hudi formats") {
@@ -350,4 +385,34 @@ class HudiScanSupportSuite extends SparkFunSuite with SharedSparkSession {
       assertHasNativeParquetScan(converted)
     }
   }
+
+  test("hudi: MOR read-optimized table scan converts to native") {
+    withTable("hudi_mor_read_optimized") {
+      spark.sql("""create table hudi_mor_read_optimized (id int, name string)
+          |using hudi
+          |tblproperties (
+          |  'hoodie.datasource.write.table.type' = 'MERGE_ON_READ'
+          |)""".stripMargin)
+      spark.sql("insert into hudi_mor_read_optimized values (1, 'v1'), (2, 'v2')")
+
+      val df = spark.read
+        .format("hudi")
+        .option("hoodie.datasource.query.type", "read_optimized")
+        .load(hudiTablePath("hudi_mor_read_optimized"))
+        .select("id", "name")
+        .orderBy("id")
+
+      logFileFormats(df)
+      val rows = df.collect().toSeq
+      assert(rows == Seq(Row(1, "v1"), Row(2, "v2")))
+      val scan = df.queryExecution.sparkPlan.collectFirst {
+        case s: org.apache.spark.sql.execution.FileSourceScanExec => s
+      }
+      assert(scan.isDefined)
+      val provider = new HudiConvertProvider
+      assert(provider.isSupported(scan.get))
+      val converted = provider.convert(scan.get)
+      assertHasNativeParquetScan(converted)
+    }
+  }
 }