You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/04 14:55:43 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6552-fe5c41aab1159ee8ea995da90b48061a905e92bf created (now 3bc2faa934)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6552-fe5c41aab1159ee8ea995da90b48061a905e92bf
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 3bc2faa934 fix: stop the leak of Arrow memory from struct outputs of the codegen dispatcher (#6552)
This branch includes the following new commits:
new 3bc2faa934 fix: stop the leak of Arrow memory from struct outputs of the codegen dispatcher (#6552)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: fix: stop the leak of Arrow memory from struct outputs of the codegen dispatcher (#6552)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6552-fe5c41aab1159ee8ea995da90b48061a905e92bf
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 3bc2faa934f04799686705b952d59ed32a707dda
Author: Peter Lee <pe...@gmail.com>
AuthorDate: Sun Oct 4 14:55:21 2026 +0000
fix: stop the leak of Arrow memory from struct outputs of the codegen dispatcher (#6552)
* fix: stop struct outputs of the codegen dispatcher from leaking Arrow memory
Arrow Java's StructVector builds its NullableStructWriter in a field
initializer, and that writer creates and allocates a child vector for every
child of the struct's field. allocateOutput constructs its struct from the
export field, which carries the children, and then calls
initializeChildrenFromFields. Under the default CONFLICT_REPLACE policy that
drops the writer's children from the struct without closing them, so their
buffers outlive the output vector. Every struct-typed output batch of the
dispatcher leaked them: 64 KiB for struct<string, int> and 80 KiB for
struct<bigint, string>.
Close the writer's children before initializeChildrenFromFields replaces them.
allocateOutput now takes the allocator, defaulting to CometArrowAllocator, so a
test can check that a closed output leaves nothing allocated.
* Write the new comments in Simplified Technical English
* Remove the -ing forms from the new test name and assert message
* Make RenamedStructVector own all of its memory
Give the StructVector constructor a field without children. Thus, the writer
creates no child vectors, and nothing must close vectors that another object
created. getField returns the export field only after
initializeChildrenFromFields. allocateOutput goes back to its original shape.
* Shorten the new comments
---
.../comet/codegen/CometBatchKernelCodegen.scala | 10 +++++--
.../codegen/CometBatchKernelCodegenOutput.scala | 31 +++++++++++++++------
.../scala/org/apache/comet/CometCodegenSuite.scala | 32 ++++++++++++++++++++++
3 files changed, 63 insertions(+), 10 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
index eadda24da7..d762e66540 100644
--- a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
+++ b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
@@ -19,6 +19,7 @@
package org.apache.comet.codegen
+import org.apache.arrow.memory.BufferAllocator
import org.apache.arrow.vector._
import org.apache.arrow.vector.complex.{ListVector, MapVector, StructVector}
import org.apache.arrow.vector.types.pojo.Field
@@ -28,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
+import org.apache.comet.CometArrowAllocator
import org.apache.comet.shims.{CometExprTraitShim, CometTypeShim}
/**
@@ -205,8 +207,12 @@ object CometBatchKernelCodegen extends Logging with CometExprTraitShim with Come
* Allocate an Arrow output vector from a pre-built `Field`. Forwards to
* [[CometBatchKernelCodegenOutput.allocateOutput]].
*/
- def allocateOutput(field: Field, numRows: Int, estimatedBytes: Int): FieldVector =
- CometBatchKernelCodegenOutput.allocateOutput(field, numRows, estimatedBytes)
+ def allocateOutput(
+ field: Field,
+ numRows: Int,
+ estimatedBytes: Int,
+ allocator: BufferAllocator = CometArrowAllocator): FieldVector =
+ CometBatchKernelCodegenOutput.allocateOutput(field, numRows, estimatedBytes, allocator)
/**
* Spark `DataType` to an Arrow `Field`, resolving mismatches between Arrow Java's default field
diff --git a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
index 33e6c0c035..2d2c8f1b2d 100644
--- a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
+++ b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
@@ -30,7 +30,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.comet.util.Utils
import org.apache.spark.sql.types._
-import org.apache.comet.CometArrowAllocator
import org.apache.comet.shims.CometTypeShim
/**
@@ -86,22 +85,26 @@ private[codegen] object CometBatchKernelCodegenOutput extends CometTypeShim {
*
* Closes the vector on any failure so a partially-initialized tree doesn't leak buffers.
*/
- def allocateOutput(field: Field, numRows: Int, estimatedBytes: Int): FieldVector = {
+ def allocateOutput(
+ field: Field,
+ numRows: Int,
+ estimatedBytes: Int,
+ allocator: BufferAllocator): FieldVector = {
val vec: FieldVector = field.getType match {
case _: ArrowType.List | _: ArrowType.LargeList | _: ArrowType.FixedSizeList =>
- val v = new RenamedListVector(field, CometArrowAllocator)
+ val v = new RenamedListVector(field, allocator)
v.initializeChildrenFromFields(field.getChildren)
v
case _: ArrowType.Map =>
- val v = new RenamedMapVector(field, CometArrowAllocator)
+ val v = new RenamedMapVector(field, allocator)
v.initializeChildrenFromFields(field.getChildren)
v
case _: ArrowType.Struct =>
- val v = new RenamedStructVector(field, CometArrowAllocator)
+ val v = new RenamedStructVector(field, allocator)
v.initializeChildrenFromFields(field.getChildren)
v
case _ =>
- field.createVector(CometArrowAllocator).asInstanceOf[FieldVector]
+ field.createVector(allocator).asInstanceOf[FieldVector]
}
try {
vec.setInitialCapacity(numRows)
@@ -138,9 +141,21 @@ private[codegen] object CometBatchKernelCodegenOutput extends CometTypeShim {
override def getField: Field = exportField
}
+ /**
+ * StructVector gets a field without children, so its writer creates no children that
+ * initializeChildrenFromFields then drops. `getField` returns `exportField` after that call.
+ */
private final class RenamedStructVector(exportField: Field, allocator: BufferAllocator)
- extends StructVector(exportField, allocator, null) {
- override def getField: Field = exportField
+ extends StructVector(exportField.getName, allocator, exportField.getFieldType, null) {
+ // False while the StructVector constructor runs.
+ private var childrenInitialized = false
+
+ override def initializeChildrenFromFields(children: java.util.List[Field]): Unit = {
+ super.initializeChildrenFromFields(children)
+ childrenInitialized = true
+ }
+
+ override def getField: Field = if (childrenInitialized) exportField else super.getField
}
/**
diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala
index fd3384a352..c9371fbf6d 100644
--- a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala
@@ -267,6 +267,38 @@ class CometCodegenSuite
}
}
+ test("a closed allocateOutput vector releases all of its memory") {
+ // Struct outputs leaked the children that StructVector's writer allocates in its
+ // constructor. The List and Map cases make sure that these outputs do not start to leak.
+ val pair = StructType(
+ Seq(StructField("name", StringType), StructField("age", IntegerType, nullable = false)))
+ val outputTypes = Seq(
+ pair,
+ StructType(
+ Seq(StructField("_1", LongType, nullable = false), StructField("_2", StringType))),
+ StructType(
+ Seq(
+ StructField("inner", pair),
+ StructField("tags", ArrayType(StringType)),
+ StructField("attrs", MapType(StringType, IntegerType)))),
+ ArrayType(pair),
+ MapType(StringType, pair),
+ StringType)
+ outputTypes.foreach { dataType =>
+ val field = CometBatchKernelCodegen.toFfiArrowField("out", dataType, nullable = true)
+ val allocator =
+ CometArrowAllocator.newChildAllocator(s"allocateOutput($dataType)", 0, Long.MaxValue)
+ try {
+ CometBatchKernelCodegen.allocateOutput(field, 4, 0, allocator).close()
+ assert(
+ allocator.getAllocatedMemory == 0,
+ s"the $dataType output did not release all of its memory")
+ } finally {
+ allocator.close()
+ }
+ }
+ }
+
test("ScalaUDF over concat(c1, c2) suppresses the null short-circuit") {
// Concat is not NullIntolerant. The dispatcher's short-circuit guard inspects every node in
// the bound tree and must skip the whole-tree null short-circuit because one child is
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org