You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by yu...@apache.org on 2026/04/14 07:29:14 UTC
(fluss) branch main updated: [spark] Fix column projection on log/upsert read path (#3057)
This is an automated email from the ASF dual-hosted git repository.
yuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new dcf055aac [spark] Fix column projection on log/upsert read path (#3057)
dcf055aac is described below
commit dcf055aac7e0604b74ed2d24e3818528e779c550
Author: Anton Borisov <14...@users.noreply.github.com>
AuthorDate: Tue Apr 14 08:29:08 2026 +0100
[spark] Fix column projection on log/upsert read path (#3057)
---
.../spark/read/FlussAppendPartitionReader.scala | 3 ++
.../fluss/spark/read/FlussPartitionReader.scala | 4 +-
.../spark/read/FlussUpsertPartitionReader.scala | 16 ++++----
.../apache/fluss/spark/SparkLogTableReadTest.scala | 36 +++++++++++++++++
.../fluss/spark/SparkPrimaryKeyTableReadTest.scala | 47 ++++++++++++++++++++++
5 files changed, 98 insertions(+), 8 deletions(-)
diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
index 520d6ae71..02f4b48dd 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussAppendPartitionReader.scala
@@ -20,6 +20,7 @@ package org.apache.fluss.spark.read
import org.apache.fluss.client.table.scanner.ScanRecord
import org.apache.fluss.config.Configuration
import org.apache.fluss.metadata.{TableBucket, TablePath}
+import org.apache.fluss.types.RowType
/** Partition reader that reads log data from a single Fluss table bucket. */
class FlussAppendPartitionReader(
@@ -29,6 +30,8 @@ class FlussAppendPartitionReader(
flussConfig: Configuration)
extends FlussPartitionReader(tablePath, flussConfig) {
+ override protected lazy val projectedRowType: RowType = rowType.project(projection)
+
private val tableBucket: TableBucket = flussPartition.tableBucket
private val partitionId = tableBucket.getPartitionId
private val bucketId = tableBucket.getBucket
diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
index 07b6a47f4..261be08a7 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussPartitionReader.scala
@@ -69,7 +69,9 @@ abstract class FlussPartitionReader(tablePath: TablePath, flussConfig: Configura
convertToSparkRow(scanRecord.getRow)
}
+ protected def projectedRowType: RowType
+
protected def convertToSparkRow(flussRow: FlussInternalRow): InternalRow = {
- DataConverter.toSparkInternalRow(flussRow, rowType)
+ DataConverter.toSparkInternalRow(flussRow, projectedRowType)
}
}
diff --git a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
index 73f3fca52..32e20e3f9 100644
--- a/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
+++ b/fluss-spark/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussUpsertPartitionReader.scala
@@ -24,7 +24,7 @@ import org.apache.fluss.config.Configuration
import org.apache.fluss.memory.MemorySegment
import org.apache.fluss.metadata.{TableBucket, TablePath}
import org.apache.fluss.record.LogRecord
-import org.apache.fluss.row.{encode, InternalRow, KeyValueRow}
+import org.apache.fluss.row.{encode, InternalRow => FlussInternalRow, KeyValueRow}
import org.apache.fluss.spark.SparkFlussConf
import org.apache.fluss.spark.utils.LogChangesIterator
import org.apache.fluss.types.{DataField, RowType}
@@ -51,6 +51,8 @@ class FlussUpsertPartitionReader(
extends FlussPartitionReader(tablePath, flussConfig)
with Logging {
+ override protected lazy val projectedRowType: RowType = rowType.project(projectionWithPks)
+
private val readOptimized = flussConfig.get(SparkFlussConf.READ_OPTIMIZED_OPTION)
private val tableBucket: TableBucket = flussPartition.tableBucket
private val snapshotId: Long = flussPartition.snapshotId
@@ -79,7 +81,7 @@ class FlussUpsertPartitionReader(
private var snapshotScanner: BatchScanner = _
private var logScanner: LogScanner = _
- private var mergedIterator: Iterator[InternalRow] = _
+ private var mergedIterator: Iterator[FlussInternalRow] = _
// initialize scanners
initialize()
@@ -111,8 +113,8 @@ class FlussUpsertPartitionReader(
tableInfo.isDefaultBucketKey)
// Create comparators based on primary key
- val comparator = new Comparator[InternalRow] {
- override def compare(o1: InternalRow, o2: InternalRow): Int = {
+ val comparator = new Comparator[FlussInternalRow] {
+ override def compare(o1: FlussInternalRow, o2: FlussInternalRow): Int = {
val key1 = keyEncoder.encodeKey(o1)
val key2 = keyEncoder.encodeKey(o2)
MemorySegment.wrap(key1).compare(MemorySegment.wrap(key2), 0, 0, key1.length)
@@ -160,7 +162,7 @@ class FlussUpsertPartitionReader(
// Convert snapshot iterator to LogRecord iterator for SortMergeReader
new CloseableIterator[LogRecord] {
- private var currentBatch: java.util.Iterator[InternalRow] = _
+ private var currentBatch: java.util.Iterator[FlussInternalRow] = _
private var hasMoreBatches = true
override def hasNext: Boolean = {
@@ -200,9 +202,9 @@ class FlussUpsertPartitionReader(
createSnapshotIterator()
}
- // Create the SortMergeReader
+ // null: scanners already project rows; passing projectionWithPks here double-projects
val sortMergeReader = new SortMergeReader(
- projectionWithPks,
+ null,
pkProjection,
snapshotIterators,
comparator,
diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
index 5f9613cbc..28b539b89 100644
--- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
+++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkLogTableReadTest.scala
@@ -211,6 +211,42 @@ class SparkLogTableReadTest extends FlussSparkTestBase {
}
}
+ test("Spark Read: log table projection with type-dependent columns") {
+ withTable("t") {
+ sql(s"""
+ |CREATE TABLE $DEFAULT_DATABASE.t (
+ |id INT,
+ |ts TIMESTAMP,
+ |name STRING,
+ |arr ARRAY<INT>,
+ |struct_col STRUCT<col1: INT, col2: STRING>,
+ |ts_ltz TIMESTAMP_LTZ
+ |)""".stripMargin)
+
+ sql(s"""
+ |INSERT INTO $DEFAULT_DATABASE.t VALUES
+ |(1, TIMESTAMP "2026-01-01 12:00:00", "a", ARRAY(1, 2), STRUCT(10, 'x'),
+ | TIMESTAMP "2026-01-01 12:00:00"),
+ |(2, TIMESTAMP "2026-01-02 12:00:00", "b", ARRAY(3, 4), STRUCT(20, 'y'),
+ | TIMESTAMP "2026-01-02 12:00:00")
+ |""".stripMargin)
+
+ // Projection reorders type-dependent columns (array, timestamp, struct)
+ checkAnswer(
+ sql(s"SELECT arr, ts, struct_col FROM $DEFAULT_DATABASE.t ORDER BY ts"),
+ Row(Seq(1, 2), java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), Row(10, "x")) ::
+ Row(Seq(3, 4), java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Row(20, "y")) :: Nil
+ )
+
+ // Projection with timestamp_ltz at shifted ordinal
+ checkAnswer(
+ sql(s"SELECT ts_ltz, name FROM $DEFAULT_DATABASE.t ORDER BY name"),
+ Row(java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), "a") ::
+ Row(java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), "b") :: Nil
+ )
+ }
+ }
+
test("Spark Read: nested data types table") {
withTable("t") {
// TODO: support map type
diff --git a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
index e75367f07..8f3faeea4 100644
--- a/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
+++ b/fluss-spark/fluss-spark-ut/src/test/scala/org/apache/fluss/spark/SparkPrimaryKeyTableReadTest.scala
@@ -255,6 +255,53 @@ class SparkPrimaryKeyTableReadTest extends FlussSparkTestBase {
}
}
+ test("Spark Read: primary key table projection with type-dependent columns") {
+ withTable("t") {
+ val tablePath = createTablePath("t")
+ sql(s"""
+ |CREATE TABLE $DEFAULT_DATABASE.t (
+ |pk INT,
+ |ts TIMESTAMP,
+ |name STRING,
+ |arr ARRAY<INT>,
+ |struct_col STRUCT<col1: INT, col2: STRING>,
+ |ts_ltz TIMESTAMP_LTZ
+ |) TBLPROPERTIES("primary.key" = "pk", "bucket.num" = 1)
+ |""".stripMargin)
+
+ sql(s"""
+ |INSERT INTO $DEFAULT_DATABASE.t VALUES
+ |(1, TIMESTAMP "2026-01-01 12:00:00", "a", ARRAY(1, 2), STRUCT(10, 'x'),
+ | TIMESTAMP "2026-01-01 12:00:00"),
+ |(2, TIMESTAMP "2026-01-02 12:00:00", "b", ARRAY(3, 4), STRUCT(20, 'y'),
+ | TIMESTAMP "2026-01-02 12:00:00")
+ |""".stripMargin)
+
+ // Log-only: projection reorders type-dependent columns (PK not in projection)
+ checkAnswer(
+ sql(s"SELECT arr, ts, struct_col FROM $DEFAULT_DATABASE.t ORDER BY ts"),
+ Row(Seq(1, 2), java.sql.Timestamp.valueOf("2026-01-01 12:00:00"), Row(10, "x")) ::
+ Row(Seq(3, 4), java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Row(20, "y")) :: Nil
+ )
+
+ // Trigger snapshot, then test with snapshot + log merge
+ flussServer.triggerAndWaitSnapshot(tablePath)
+
+ sql(s"""
+ |INSERT INTO $DEFAULT_DATABASE.t VALUES
+ |(1, TIMESTAMP "2026-03-01 12:00:00", "a_updated", ARRAY(10, 20), STRUCT(100, 'xx'),
+ | TIMESTAMP "2026-03-01 12:00:00")
+ |""".stripMargin)
+
+ // Snapshot + log: projection with type-dependent columns at shifted ordinals
+ checkAnswer(
+ sql(s"SELECT ts_ltz, arr, name FROM $DEFAULT_DATABASE.t ORDER BY name"),
+ Row(java.sql.Timestamp.valueOf("2026-03-01 12:00:00"), Seq(10, 20), "a_updated") ::
+ Row(java.sql.Timestamp.valueOf("2026-01-02 12:00:00"), Seq(3, 4), "b") :: Nil
+ )
+ }
+ }
+
private def genInputPartition(
tablePath: TablePath,
partitionName: String): Array[FlussUpsertInputPartition] = {