You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/08/26 14:12:08 UTC

(paimon) branch master updated: [spark] Fix bootstrap record reader not close (#4061)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 25a4f9958 [spark] Fix bootstrap record reader not close (#4061)
25a4f9958 is described below

commit 25a4f9958f566e723129fcb905dbf1b1dc116a28
Author: Xiduo You <ul...@gmail.com>
AuthorDate: Mon Aug 26 22:12:03 2024 +0800

    [spark] Fix bootstrap record reader not close (#4061)
---
 .../scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
index dddbaf24b..391ba2b87 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
@@ -343,10 +343,12 @@ case class PaimonSparkWriter(table: FileStoreTable) {
             val rowProject = CodeGenUtils.newProjection(rowType, primaryKeys)
             val bootstrapSer = InternalSerializers.create(bootstrapType)
             val rowSer = InternalSerializers.create(rowType)
-            new IndexBootstrap(table)
+            val bootstrapIterator = new IndexBootstrap(table)
               .bootstrap(numSparkPartitions, sparkPartitionId)
               .toCloseableIterator
-              .asScala
+            TaskContext.get().addTaskCompletionListener[Unit](_ => bootstrapIterator.close())
+
+            bootstrapIterator.asScala
               .map(
                 row => {
                   val bytes: Array[Byte] =