You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "L. C. Hsieh (Jira)" <ji...@apache.org> on 2021/07/16 07:42:00 UTC
[jira] [Assigned] (SPARK-35972) When replace ExtractValue in
NestedColumnAliasing we should use semanticEquals
[ https://issues.apache.org/jira/browse/SPARK-35972?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
L. C. Hsieh reassigned SPARK-35972:
-----------------------------------
Assignee: angerszhu
> When replace ExtractValue in NestedColumnAliasing we should use semanticEquals
> ------------------------------------------------------------------------------
>
> Key: SPARK-35972
> URL: https://issues.apache.org/jira/browse/SPARK-35972
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.1.2, 3.2.0
> Reporter: angerszhu
> Assignee: angerszhu
> Priority: Major
> Fix For: 3.2.0
>
>
> {code:java}
> Job aborted due to stage failure: Task 47 in stage 1.0 failed 4 times, most recent failure: Lost task 47.3 in stage 1.0 (TID 328) (ip-idata-server.shopee.io executor 3): org.apache.spark.sql.catalyst.errors.package$TreeNodeException: Binding attribute, tree: _gen_alias_788#788
> at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:75)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:74)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$1(TreeNode.scala:318)
> at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:74)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:318)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChild$2(TreeNode.scala:377)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$4(TreeNode.scala:438)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:438)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChild$2(TreeNode.scala:377)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$4(TreeNode.scala:438)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:438)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChild$2(TreeNode.scala:386)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$4(TreeNode.scala:438)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:438)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChild$2(TreeNode.scala:377)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$4(TreeNode.scala:438)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:438)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChild$2(TreeNode.scala:386)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$4(TreeNode.scala:438)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:438)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
> at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:323)
> at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:307)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$.bindReference(BoundAttribute.scala:74)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$.$anonfun$bindReferences$1(BoundAttribute.scala:96)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at scala.collection.TraversableLike.map(TraversableLike.scala:238)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> at scala.collection.immutable.List.map(List.scala:298)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$.bindReferences(BoundAttribute.scala:96)
> at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:160)
> at org.apache.spark.sql.execution.ProjectExec.$anonfun$doExecute$1(basicPhysicalOperators.scala:93)
> at org.apache.spark.sql.execution.ProjectExec.$anonfun$doExecute$1$adapted(basicPhysicalOperators.scala:92)
> at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndexInternal$2(RDD.scala:885)
> at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsWithIndexInternal$2$adapted(RDD.scala:885)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)
> at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)
> at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)
> at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)
> at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)
> at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:335)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
> at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
> at org.apache.spark.scheduler.Task.run(Task.scala:131)
> at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: Couldn't find _gen_alias_788#788 in [view_event_id#56,view_timestamp#57,page_type#82,page_section#58,target_type#88,platform#83,ab_test#65,event_timestamp#59,user_id#60L,device_id#61,app_version#67,session_id#86,pre_view_event_id#62,rand_pre_view_event_id#63,_gen_alias_726#726,_gen_alias_727#727,_gen_alias_728#728,_gen_alias_729#729,_gen_alias_730#730,_gen_alias_731#731,_gen_alias_732#732,_gen_alias_733#733,_gen_alias_734#734,_gen_alias_735#735,_gen_alias_736#736,_gen_alias_737#737,_gen_alias_738#738,_gen_alias_739#739,_gen_alias_740#740,_gen_alias_741#741,_gen_alias_742#742,_gen_alias_743#743,_gen_alias_744#744,_gen_alias_745#745,_gen_alias_746#746,_gen_alias_747#747,_gen_alias_748#748,_gen_alias_749#749,_gen_alias_750#750,_gen_alias_751#751,_gen_alias_752#752,_gen_alias_753#753,_gen_alias_754#754,_gen_alias_755#755,_gen_alias_756#756,_gen_alias_757#757,_gen_alias_758#758,_gen_alias_759#759,_gen_alias_760#760,_gen_alias_761#761,_gen_alias_762#762,_gen_alias_763#763,_gen_alias_764#764,_gen_alias_765#765,_gen_alias_766#766,_gen_alias_767#767,_gen_alias_768#768,_gen_alias_769#769,_gen_alias_770#770,_gen_alias_771#771,_gen_alias_772#772,_gen_alias_773#773,_gen_alias_774#774,_gen_alias_775#775,_gen_alias_776#776,_gen_alias_777#777,_gen_alias_778#778,_gen_alias_779#779,_gen_alias_780#780,_gen_alias_781#781,_gen_alias_782#782,_gen_alias_783#783,_gen_alias_784#784,_gen_alias_785#785,_gen_alias_786#786,_gen_alias_787#787]
> at scala.sys.package$.error(package.scala:30)
> at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.$anonfun$applyOrElse$1(BoundAttribute.scala:81)
> at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
> ... 157 more
> Driver stacktrace:
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org