You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/08/26 14:12:08 UTC
(paimon) branch master updated: [spark] Fix bootstrap record reader not close (#4061)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 25a4f9958 [spark] Fix bootstrap record reader not close (#4061)
25a4f9958 is described below
commit 25a4f9958f566e723129fcb905dbf1b1dc116a28
Author: Xiduo You <ul...@gmail.com>
AuthorDate: Mon Aug 26 22:12:03 2024 +0800
[spark] Fix bootstrap record reader not close (#4061)
---
.../scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
index dddbaf24b..391ba2b87 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
@@ -343,10 +343,12 @@ case class PaimonSparkWriter(table: FileStoreTable) {
val rowProject = CodeGenUtils.newProjection(rowType, primaryKeys)
val bootstrapSer = InternalSerializers.create(bootstrapType)
val rowSer = InternalSerializers.create(rowType)
- new IndexBootstrap(table)
+ val bootstrapIterator = new IndexBootstrap(table)
.bootstrap(numSparkPartitions, sparkPartitionId)
.toCloseableIterator
- .asScala
+ TaskContext.get().addTaskCompletionListener[Unit](_ => bootstrapIterator.close())
+
+ bootstrapIterator.asScala
.map(
row => {
val bytes: Array[Byte] =