You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Yi Zhu (Jira)" <ji...@apache.org> on 2024/06/28 09:06:00 UTC
[jira] [Updated] (SPARK-48750) AQEPropagateEmptyRelation convert broadcast query stage plan to empty relation causing error
[ https://issues.apache.org/jira/browse/SPARK-48750?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yi Zhu updated SPARK-48750:
---------------------------
Description:
{code:java}
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.util.concurrent.ExecutionException: Boxed Error
at scala.concurrent.impl.Promise$.resolver(Promise.scala:87)
at scala.concurrent.impl.Promise$.scala$concurrent$impl$Promise$$resolveTry(Promise.scala:79)
at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
at scala.concurrent.Promise.complete(Promise.scala:53)
at scala.concurrent.Promise.complete$(Promise.scala:52)
at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:187)
at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:208)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.$anonfun$doExecuteBroadcast$1(AdaptiveSparkPlanExec.scala:369)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.withFinalPlanUpdate(AdaptiveSparkPlanExec.scala:376)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.doExecuteBroadcast(AdaptiveSparkPlanExec.scala:364)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:197)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:222)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:219)
at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:193)
at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$2(SubqueryBroadcastExec.scala:81)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withExecutionId$1(SQLExecution.scala:146)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:170)
at org.apache.spark.sql.execution.SQLExecution$.withExecutionId(SQLExecution.scala:144)
at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$1(SubqueryBroadcastExec.scala:78)
at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
at scala.util.Success.$anonfun$map$1(Try.scala:255)
at scala.util.Success.map(Try.scala:213)
at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
... 5 more
{code}
{code:java}
=== Applying Rule org.apache.spark.sql.execution.adaptive.AQEPropagateEmptyRelation ===
Project [bm_id#1469, uid#1491L, ... 4 more fields] Project [bm_id#1469, uid#1491L, ... 4 more fields]
!+- Join Inner, (uid#1491L = uid#1495L) +- LocalRelation <empty>, [bm_id#1469, uid#1491L, ... 5 more fields]
! :- LogicalQueryStage Project [xx#1469, uid#1491L, ... 4 more fields], BroadcastQueryStage 1
! +- Project [uid#1495L]
! +- Filter xxxx
! +- Relation db.table[uid#1495L,request_id#1496,... 40 more fields] parquet {code}
was:
{code:java}
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.util.concurrent.ExecutionException: Boxed Error
at scala.concurrent.impl.Promise$.resolver(Promise.scala:87)
at scala.concurrent.impl.Promise$.scala$concurrent$impl$Promise$$resolveTry(Promise.scala:79)
at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
at scala.concurrent.Promise.complete(Promise.scala:53)
at scala.concurrent.Promise.complete$(Promise.scala:52)
at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:187)
at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: assertion failed
at scala.Predef$.assert(Predef.scala:208)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.$anonfun$doExecuteBroadcast$1(AdaptiveSparkPlanExec.scala:369)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.withFinalPlanUpdate(AdaptiveSparkPlanExec.scala:376)
at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.doExecuteBroadcast(AdaptiveSparkPlanExec.scala:364)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:197)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:222)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:219)
at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:193)
at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$2(SubqueryBroadcastExec.scala:81)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withExecutionId$1(SQLExecution.scala:146)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:170)
at org.apache.spark.sql.execution.SQLExecution$.withExecutionId(SQLExecution.scala:144)
at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$1(SubqueryBroadcastExec.scala:78)
at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
at scala.util.Success.$anonfun$map$1(Try.scala:255)
at scala.util.Success.map(Try.scala:213)
at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
... 5 more
{code}
{code:java}
=== Applying Rule org.apache.spark.sql.execution.adaptive.AQEPropagateEmptyRelation ===
Project [bm_id#1469, uid#1491L, ... 4 more fields] Project [bm_id#1469, uid#1491L, ... 4 more fields]
!+- Join Inner, (uid#1491L = uid#1495L) +- LocalRelation <empty>, [bm_id#1469, uid#1491L, ... 5 more fields]
! :- LogicalQueryStage Project [bm_id#1469, uid#1491L, ... 4 more fields], BroadcastQueryStage 1
! +- Project [uid#1495L]
! +- Filter ((((((isnotnull(tz_type#1534) AND isnotnull(grass_date#1536)) AND isnotnull(lump_sum_type#1523)) AND (tz_type#1534 = local)) AND (grass_date#1536 = 2024-06-11)) AND (lump_sum_type#1523 = 2)) AND isnotnull(uid#1495L))
! +- Relation sbs_mart.ods_shopee_pms_db__bm_version_tab_df_cn[uid#1495L,request_id#1496,... 40 more fields] parquet {code}
> AQEPropagateEmptyRelation convert broadcast query stage plan to empty relation causing error
> --------------------------------------------------------------------------------------------
>
> Key: SPARK-48750
> URL: https://issues.apache.org/jira/browse/SPARK-48750
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.2.4, 3.5.1, 3.3.4
> Reporter: Yi Zhu
> Priority: Major
>
> {code:java}
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> Caused by: java.util.concurrent.ExecutionException: Boxed Error
> at scala.concurrent.impl.Promise$.resolver(Promise.scala:87)
> at scala.concurrent.impl.Promise$.scala$concurrent$impl$Promise$$resolveTry(Promise.scala:79)
> at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
> at scala.concurrent.Promise.complete(Promise.scala:53)
> at scala.concurrent.Promise.complete$(Promise.scala:52)
> at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:187)
> at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
> at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.AssertionError: assertion failed
> at scala.Predef$.assert(Predef.scala:208)
> at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.$anonfun$doExecuteBroadcast$1(AdaptiveSparkPlanExec.scala:369)
> at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.withFinalPlanUpdate(AdaptiveSparkPlanExec.scala:376)
> at org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.doExecuteBroadcast(AdaptiveSparkPlanExec.scala:364)
> at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(SparkPlan.scala:197)
> at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:222)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:219)
> at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(SparkPlan.scala:193)
> at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$2(SubqueryBroadcastExec.scala:81)
> at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withExecutionId$1(SQLExecution.scala:146)
> at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:170)
> at org.apache.spark.sql.execution.SQLExecution$.withExecutionId(SQLExecution.scala:144)
> at org.apache.spark.sql.execution.SubqueryBroadcastExec.$anonfun$relationFuture$1(SubqueryBroadcastExec.scala:78)
> at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
> at scala.util.Success.$anonfun$map$1(Try.scala:255)
> at scala.util.Success.map(Try.scala:213)
> at scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
> at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
> ... 5 more
> {code}
>
>
> {code:java}
> === Applying Rule org.apache.spark.sql.execution.adaptive.AQEPropagateEmptyRelation ===
> Project [bm_id#1469, uid#1491L, ... 4 more fields] Project [bm_id#1469, uid#1491L, ... 4 more fields]
> !+- Join Inner, (uid#1491L = uid#1495L) +- LocalRelation <empty>, [bm_id#1469, uid#1491L, ... 5 more fields]
> ! :- LogicalQueryStage Project [xx#1469, uid#1491L, ... 4 more fields], BroadcastQueryStage 1
> ! +- Project [uid#1495L]
> ! +- Filter xxxx
> ! +- Relation db.table[uid#1495L,request_id#1496,... 40 more fields] parquet {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org