You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Yang Jie (Jira)" <ji...@apache.org> on 2025/04/01 07:33:00 UTC
[jira] [Updated] (SPARK-51677) Memleak in connect-jvm-client tests

     [ https://issues.apache.org/jira/browse/SPARK-51677?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Yang Jie updated SPARK-51677:
-----------------------------
    Description: 
build/sbt "connect-client-jvm/test" -Phive -Dspark.debug.sc.jvm.client=true

 
{code:java}
[info] - SPARK-50965: Multiple named and positional parameterized nodes in the parsed logical plan (41 milliseconds) java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (12192) Allocator(ROOT) 0/12192/6325056/9223372036854775807 (res/actual/peak/limit) at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504) at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27) at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632) at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224) at org.apache.spark.sql.connect.ClientE2ETestSuite.afterAll(ClientE2ETestSuite.scala:51) at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225) at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377) at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373) at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962) at org.scalatest.Status.withAfterEffect(Status.scala:373) at org.scalatest.Status.withAfterEffect$(Status.scala:371) at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863) at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224) at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) at org.apache.spark.sql.connect.ClientE2ETestSuite.run(ClientE2ETestSuite.scala:51) at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840)
...
[info] - executeCommand (24 milliseconds)
java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (128000)
Allocator(ROOT) 0/128000/128032/9223372036854775807 (res/actual/peak/limit)


	at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504)
	at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27)
	at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632)
	at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783)
	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226)
	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224)
	at org.apache.spark.sql.connect.SparkSessionE2ESuite.afterAll(SparkSessionE2ESuite.scala:38)
	at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225)
	at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377)
	at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373)
	at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962)
	at org.scalatest.Status.withAfterEffect(Status.scala:373)
	at org.scalatest.Status.withAfterEffect$(Status.scala:371)
	at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863)
	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224)
	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
	at org.apache.spark.sql.connect.SparkSessionE2ESuite.run(SparkSessionE2ESuite.scala:38)
	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321)
	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517)
	at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840) {code}

  was:
build/sbt "connect-client-jvm/test" -Phive -Dspark.debug.sc.jvm.client=true

 
{code:java}
[info] - SPARK-50965: Multiple named and positional parameterized nodes in the parsed logical plan (41 milliseconds) java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (12192) Allocator(ROOT) 0/12192/6325056/9223372036854775807 (res/actual/peak/limit) at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504) at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27) at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632) at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224) at org.apache.spark.sql.connect.ClientE2ETestSuite.afterAll(ClientE2ETestSuite.scala:51) at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225) at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377) at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373) at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962) at org.scalatest.Status.withAfterEffect(Status.scala:373) at org.scalatest.Status.withAfterEffect$(Status.scala:371) at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863) at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224) at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) at org.apache.spark.sql.connect.ClientE2ETestSuite.run(ClientE2ETestSuite.scala:51) at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840)

[info] - executeCommand (24 milliseconds)
java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (128000)
Allocator(ROOT) 0/128000/128032/9223372036854775807 (res/actual/peak/limit)


	at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504)
	at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27)
	at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632)
	at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783)
	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226)
	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224)
	at org.apache.spark.sql.connect.SparkSessionE2ESuite.afterAll(SparkSessionE2ESuite.scala:38)
	at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225)
	at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377)
	at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373)
	at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962)
	at org.scalatest.Status.withAfterEffect(Status.scala:373)
	at org.scalatest.Status.withAfterEffect$(Status.scala:371)
	at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863)
	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224)
	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
	at org.apache.spark.sql.connect.SparkSessionE2ESuite.run(SparkSessionE2ESuite.scala:38)
	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321)
	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517)
	at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840) {code}


> Memleak in connect-jvm-client tests
> -----------------------------------
>
>                 Key: SPARK-51677
>                 URL: https://issues.apache.org/jira/browse/SPARK-51677
>             Project: Spark
>          Issue Type: Bug
>          Components: Connect, Tests
>    Affects Versions: 4.0.0, 4.1.0
>            Reporter: Yang Jie
>            Priority: Major
>
> build/sbt "connect-client-jvm/test" -Phive -Dspark.debug.sc.jvm.client=true
>  
> {code:java}
> [info] - SPARK-50965: Multiple named and positional parameterized nodes in the parsed logical plan (41 milliseconds) java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (12192) Allocator(ROOT) 0/12192/6325056/9223372036854775807 (res/actual/peak/limit) at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504) at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27) at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632) at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226) at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224) at org.apache.spark.sql.connect.ClientE2ETestSuite.afterAll(ClientE2ETestSuite.scala:51) at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225) at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377) at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373) at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962) at org.scalatest.Status.withAfterEffect(Status.scala:373) at org.scalatest.Status.withAfterEffect$(Status.scala:371) at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863) at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224) at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) at org.apache.spark.sql.connect.ClientE2ETestSuite.run(ClientE2ETestSuite.scala:51) at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321) at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517) at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840)
> ...
> [info] - executeCommand (24 milliseconds)
> java.lang.IllegalStateException: Memory was leaked by query. Memory leaked: (128000)
> Allocator(ROOT) 0/128000/128032/9223372036854775807 (res/actual/peak/limit)
> 	at org.apache.arrow.memory.BaseAllocator.close(BaseAllocator.java:504)
> 	at org.apache.arrow.memory.RootAllocator.close(RootAllocator.java:27)
> 	at org.apache.spark.sql.connect.SparkSession.close(SparkSession.scala:632)
> 	at org.apache.spark.sql.SparkSession.stop(SparkSession.scala:783)
> 	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll(RemoteSparkSession.scala:226)
> 	at org.apache.spark.sql.connect.test.RemoteSparkSession.afterAll$(RemoteSparkSession.scala:224)
> 	at org.apache.spark.sql.connect.SparkSessionE2ESuite.afterAll(SparkSessionE2ESuite.scala:38)
> 	at org.scalatest.BeforeAndAfterAll.$anonfun$run$1(BeforeAndAfterAll.scala:225)
> 	at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377)
> 	at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373)
> 	at org.scalatest.CompositeStatus.whenCompleted(Status.scala:962)
> 	at org.scalatest.Status.withAfterEffect(Status.scala:373)
> 	at org.scalatest.Status.withAfterEffect$(Status.scala:371)
> 	at org.scalatest.CompositeStatus.withAfterEffect(Status.scala:863)
> 	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:224)
> 	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> 	at org.apache.spark.sql.connect.SparkSessionE2ESuite.run(SparkSessionE2ESuite.scala:38)
> 	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321)
> 	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517)
> 	at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414)
> 	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> 	at java.base/java.lang.Thread.run(Thread.java:840) {code}



--
This message was sent by Atlassian Jira
(v8.20.10#820010)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org