You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by yu...@apache.org on 2026/04/14 07:29:14 UTC

(fluss) branch main updated: [spark] Fix column projection on log/upsert read path (#3057)

This is an automated email from the ASF dual-hosted git repository.

yuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git


The following commit(s) were added to refs/heads/main by this push:
     new dcf055aac [spark] Fix column projection on log/upsert read path (#3057)
dcf055aac is described below

commit dcf055aac7e0604b74ed2d24e3818528e779c550
Author: Anton Borisov <14...@users.noreply.github.com>
AuthorDate: Tue Apr 14 08:29:08 2026 +0100

    [spark] Fix column projection on log/upsert read path (#3057)
---
 .../spark/read/FlussAppendPartitionReader.scala    |  3 ++
 .../fluss/spark/read/FlussPartitionReader.scala    |  4 +-
 .../spark/read/FlussUpsertPartitionReader.scala    | 16 ++++----
 .../apache/fluss/spark/SparkLogTableReadTest.scala | 36 +++++++++++++++++
 .../fluss/spark/SparkPrimaryKeyTableReadTest.scala | 47 ++++++++++++++++++++++
 5 files changed, 98 insertions(+), 8 deletions(-)

diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
index 520d6ae71..02f4b48dd 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
@@ -20,6 +20,7 @@ package org.apache.fluss.spark.read
 import org.apache.fluss.client.table.scanner.ScanRecord
 import org.apache.fluss.config.Configuration
 import org.apache.fluss.metadata.{TableBucket, TablePath}
+import org.apache.fluss.types.RowType
 
 /** Partition reader that reads log data from a single Fluss table bucket. */
 class FlussAppendPartitionReader(
@@ -29,6 +30,8 @@ class FlussAppendPartitionReader(
     flussConfig: Configuration)
   extends FlussPartitionReader(tablePath, flussConfig) {
 
+  override protected lazy val projectedRowType: RowType = rowType.project(projection)
+
   private val tableBucket: TableBucket = flussPartition.tableBucket
   private val partitionId = tableBucket.getPartitionId
   private val bucketId = tableBucket.getBucket
diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
index 07b6a47f4..261be08a7 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
@@ -69,7 +69,9 @@ abstract class FlussPartitionReader(tablePath: TablePath, flussConfig: Configura
     convertToSparkRow(scanRecord.getRow)
   }
 
+  protected def projectedRowType: RowType
+
   protected def convertToSparkRow(flussRow: FlussInternalRow): InternalRow = {
-    DataConverter.toSparkInternalRow(flussRow, rowType)
+    DataConverter.toSparkInternalRow(flussRow, projectedRowType)
   }
 }
diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
index 73f3fca52..32e20e3f9 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
@@ -24,7 +24,7 @@ import org.apache.fluss.config.Configuration
 import org.apache.fluss.memory.MemorySegment
 import org.apache.fluss.metadata.{TableBucket, TablePath}
 import org.apache.fluss.record.LogRecord
-import org.apache.fluss.row.{encode, InternalRow, KeyValueRow}
+import org.apache.fluss.row.{encode, InternalRow => FlussInternalRow, KeyValueRow}
 import org.apache.fluss.spark.SparkFlussConf
 import org.apache.fluss.spark.utils.LogChangesIterator
 import org.apache.fluss.types.{DataField, RowType}
@@ -51,6 +51,8 @@ class FlussUpsertPartitionReader(
   extends FlussPartitionReader(tablePath, flussConfig)
   with Logging {
 
+  override protected lazy val projectedRowType: RowType = rowType.project(projectionWithPks)
+
   private val readOptimized = flussConfig.get(SparkFlussConf.READ_OPTIMIZED_OPTION)
   private val tableBucket: TableBucket = flussPartition.tableBucket
   private val snapshotId: Long = flussPartition.snapshotId
@@ -79,7 +81,7 @@ class FlussUpsertPartitionReader(
 
   private var snapshotScanner: BatchScanner = _
   private var logScanner: LogScanner = _
-  private var mergedIterator: Iterator[InternalRow] = _
+  private var mergedIterator: Iterator[FlussInternalRow] = _
 
   // initialize scanners
   initialize()
@@ -111,8 +113,8 @@ class FlussUpsertPartitionReader(
         tableInfo.isDefaultBucketKey)
 
     // Create comparators based on primary key
-    val comparator = new Comparator[InternalRow] {
-      override def compare(o1: InternalRow, o2: InternalRow): Int = {
+    val comparator = new Comparator[FlussInternalRow] {
+      override def compare(o1: FlussInternalRow, o2: FlussInternalRow): Int = {
         val key1 = keyEncoder.encodeKey(o1)
         val key2 = keyEncoder.encodeKey(o2)
         MemorySegment.wrap(key1).compare(MemorySegment.wrap(key2), 0, 0, key1.length)
@@ -160,7 +162,7 @@ class FlussUpsertPartitionReader(
 
       // Convert snapshot iterator to LogRecord iterator for SortMergeReader
       new CloseableIterator[LogRecord] {
-        private var currentBatch: java.util.Iterator[InternalRow] = _
+        private var currentBatch: java.util.Iterator[FlussInternalRow] = _
         private var hasMoreBatches = true
 
         override def hasNext: Boolean = {
@@ -200,9 +202,9 @@ class FlussUpsertPartitionReader(
       createSnapshotIterator()
     }
 
-    // Create the SortMergeReader
+    // null: scanners already project rows; passing projectionWithPks here double-projects
     val sortMergeReader = new SortMergeReader(
-      projectionWithPks,
+      null,
       pkProjection,
       snapshotIterators,
       comparator,
diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
index 5f9613cbc..28b539b89 100644
--- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
+++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
@@ -211,6 +211,42 @@ class SparkLogTableReadTest extends FlussSparkTestBase {
     }
   }
 
+  test("Spark Read: log table projection with type-dependent columns") {
+    withTable("t") {
+      sql(s"""
+             |CREATE TABLE $DEFAULT_DATABASE.t (
+             |id INT,
+             |ts TIMESTAMP,
+             |name STRING,
+             |arr ARRAY<INT>,
+             |struct_col STRUCT<col1: INT, col2: STRING>,
+             |ts_ltz TIMESTAMP_LTZ
+             |)""".stripMargin)
+
+      sql(s"""
+             |INSERT INTO $DEFAULT_DATABASE.t VALUES
+             |(1, TIMESTAMP "2026-01-01 12:00:00", "a", ARRAY(1, 2), STRUCT(10, 'x'),
+             | TIMESTAMP "2026-01-01 12:00:00"),
+             |(2, TIMESTAMP "2026-01-02 12:00:00", "b", ARRAY(3, 4), STRUCT(20, 'y'),
+             | TIMESTAMP "2026-01-02 12:00:00")
+             |""".stripMargin)
+
+      // Projection reorders type-dependent columns (array, timestamp, struct)
+      checkAnswer(
+        sql(s"SELECT arr, ts, struct_col FROM $DEFAULT_DATABASE.t ORDER BY ts"),
+        Row(Seq(1, 2), java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), Row(10, "x")) ::
+          Row(Seq(3, 4), java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Row(20, "y")) :: Nil
+      )
+
+      // Projection with timestamp_ltz at shifted ordinal
+      checkAnswer(
+        sql(s"SELECT ts_ltz, name FROM $DEFAULT_DATABASE.t ORDER BY name"),
+        Row(java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), "a") ::
+          Row(java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), "b") :: Nil
+      )
+    }
+  }
+
   test("Spark Read: nested data types table") {
     withTable("t") {
       // TODO: support map type
diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
index e75367f07..8f3faeea4 100644
--- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
+++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
@@ -255,6 +255,53 @@ class SparkPrimaryKeyTableReadTest extends FlussSparkTestBase {
     }
   }
 
+  test("Spark Read: primary key table projection with type-dependent columns") {
+    withTable("t") {
+      val tablePath = createTablePath("t")
+      sql(s"""
+             |CREATE TABLE $DEFAULT_DATABASE.t (
+             |pk INT,
+             |ts TIMESTAMP,
+             |name STRING,
+             |arr ARRAY<INT>,
+             |struct_col STRUCT<col1: INT, col2: STRING>,
+             |ts_ltz TIMESTAMP_LTZ
+             |) TBLPROPERTIES("primary.key" = "pk", "bucket.num" = 1)
+             |""".stripMargin)
+
+      sql(s"""
+             |INSERT INTO $DEFAULT_DATABASE.t VALUES
+             |(1, TIMESTAMP "2026-01-01 12:00:00", "a", ARRAY(1, 2), STRUCT(10, 'x'),
+             | TIMESTAMP "2026-01-01 12:00:00"),
+             |(2, TIMESTAMP "2026-01-02 12:00:00", "b", ARRAY(3, 4), STRUCT(20, 'y'),
+             | TIMESTAMP "2026-01-02 12:00:00")
+             |""".stripMargin)
+
+      // Log-only: projection reorders type-dependent columns (PK not in projection)
+      checkAnswer(
+        sql(s"SELECT arr, ts, struct_col FROM $DEFAULT_DATABASE.t ORDER BY ts"),
+        Row(Seq(1, 2), java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), Row(10, "x")) ::
+          Row(Seq(3, 4), java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Row(20, "y")) :: Nil
+      )
+
+      // Trigger snapshot, then test with snapshot + log merge
+      flussServer.triggerAndWaitSnapshot(tablePath)
+
+      sql(s"""
+             |INSERT INTO $DEFAULT_DATABASE.t VALUES
+             |(1, TIMESTAMP "2026-03-01 12:00:00", "a_updated", ARRAY(10, 20), STRUCT(100, 'xx'),
+             | TIMESTAMP "2026-03-01 12:00:00")
+             |""".stripMargin)
+
+      // Snapshot + log: projection with type-dependent columns at shifted ordinals
+      checkAnswer(
+        sql(s"SELECT ts_ltz, arr, name FROM $DEFAULT_DATABASE.t ORDER BY name"),
+        Row(java.sql.Timestamp.valueOf("2026-03-01 12:00:00"), Seq(10, 20), "a_updated") ::
+          Row(java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Seq(3, 4), "b") :: Nil
+      )
+    }
+  }
+
   private def genInputPartition(
       tablePath: TablePath,
       partitionName: String): Array[FlussUpsertInputPartition] = {