You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/09/19 03:51:08 UTC
(paimon) branch master updated: [hotfix] Fix unspecified value parameter when using PaimonMetadataColumn.get method (#4211)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 29c807750 [hotfix] Fix unspecified value parameter when using PaimonMetadataColumn.get method (#4211)
29c807750 is described below
commit 29c807750b0cd78591fe9077efc1f4a4e1fdeb72
Author: Kerwin <37...@users.noreply.github.com>
AuthorDate: Thu Sep 19 11:51:04 2024 +0800
[hotfix] Fix unspecified value parameter when using PaimonMetadataColumn.get method (#4211)
---
.../src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala | 5 ++++-
.../src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
index cb43560dc..fb494a492 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
@@ -113,12 +113,15 @@ abstract class PaimonBaseScan(
inputPartitions
}
+ final def partitionType: StructType = {
+ SparkTypeUtils.toSparkPartitionType(table)
+ }
+
override def readSchema(): StructType = {
StructType(requiredTableFields ++ metadataFields)
}
override def toBatch: Batch = {
- val partitionType = SparkTypeUtils.toSparkPartitionType(table)
val metadataColumns =
metadataFields.map(field => PaimonMetadataColumn.get(field.name, partitionType))
PaimonBatch(lazyInputPartitions, readBuilder, metadataColumns)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
index c64ec7310..963d9fadd 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
@@ -75,7 +75,7 @@ case class PaimonStatistics[T <: PaimonBaseScan](scan: T) extends Statistics {
val metadataSchemaSize = scan.metadataFields.map {
field =>
- val dataField = PaimonMetadataColumn.get(field.name).toPaimonDataField
+ val dataField = PaimonMetadataColumn.get(field.name, scan.partitionType).toPaimonDataField
getSizeForField(dataField)
}.sum
val metadataSizeInBytes = paimonStats.mergedRecordCount().getAsLong * metadataSchemaSize