You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2024/12/13 18:30:07 UTC
(spark) branch master updated: [SPARK-50338][CORE] Make LazyTry exceptions less verbose
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 42ce6044e15b [SPARK-50338][CORE] Make LazyTry exceptions less verbose
42ce6044e15b is described below
commit 42ce6044e15bf09f10ee6bf507e2cbf02776764d
Author: Julek Sompolski <Juliusz Sompolski>
AuthorDate: Fri Dec 13 10:29:54 2024 -0800
[SPARK-50338][CORE] Make LazyTry exceptions less verbose
### What changes were proposed in this pull request?
For LazyTry, we want two stack traces:
1. the one from the original execution when the exception happened
2. the one from the current execution, that retrieved the exception.
The first one is added as suppressed exception. The second one is stitched into the stack trace of the rethrown exception.
The third one (`TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE`) doesn't need to be visible to users as a suppressed exception. We only need the depth of the stacktrace until `doTryWithCallerStacktrace`. Use a custom exception `OriginalTryStackTraceException` to store it together with the suppressed first one.
Because of the duplicated `TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE` exceptions it was actually also doing the stitching incorrectly.
#### Before
There are 4 suppressed exception.
```
scala> sql("select make_timestamp(2024, 1, 1, 0, 0, 0, 1)").collect()
org.apache.spark.SparkDateTimeException: [INVALID_TIMEZONE] The timezone: 1 is invalid. The timezone must be either a region-based zone ID or a zone offset. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm’ or '(+|-)HH:mm:ss', e.g '-08' , '+01:00' or '-13:33:33', and must be in the range from -18:00 to +18:00. 'Z' and 'UTC' are accepted as synonyms for '+00:00'. SQLSTATE: 22009
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1429)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:228)
at org.apache.spark.sql.execution.QueryExecution.simpleString(QueryExecution.scala:277)
at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:324)
at org.apache.spark.sql.execution.QueryExecution.explainString(QueryExecution.scala:292)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:148)
at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:268)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:124)
at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:108)
at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:107)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:124)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:291)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:123)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:77)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:233)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2199)
at org.apache.spark.sql.Dataset.collect(Dataset.scala:1473)
... 30 elided
Caused by: java.time.DateTimeException: Invalid ID for ZoneOffset, invalid format: 1
at java.base/java.time.ZoneOffset.of(ZoneOffset.java:243)
at java.base/java.time.ZoneId.of(ZoneId.java:404)
at java.base/java.time.ZoneId.of(ZoneId.java:358)
at java.base/java.time.ZoneId.of(ZoneId.java:314)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:49)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 50 more
Suppressed: java.lang.Exception: Stacktrace under doTryWithCallerStacktrace
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
Suppressed: java.lang.Exception: Full stacktrace of original doTryWithCallerStacktrace caller
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 50 more
Suppressed: java.lang.Exception: Stacktrace under doTryWithCallerStacktrace
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1429)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
Suppressed: java.lang.Exception: Full stacktrace of original doTryWithCallerStacktrace caller
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1429)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1375)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 50 more
```
#### After
There is 1 suppressed exception, showing the stack trace of the original caller that triggered the computation of the lazy (it is the same as the caller that reads the lazy, because they are read immediately, but it possibly could be different, and then both the stacktraces are needed for debugging)
```
scala> sql("select make_timestamp(2024, 1, 1, 0, 0, 0, 1)").collect()
org.apache.spark.SparkDateTimeException: [INVALID_TIMEZONE] The timezone: 1 is invalid. The timezone must be either a region-based zone ID or a zone offset. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format '(+|-)HH', '(+|-)HH:mm’ or '(+|-)HH:mm:ss', e.g '-08' , '+01:00' or '-13:33:33', and must be in the range from -18:00 to +18:00. 'Z' and 'UTC' are accepted as synonyms for '+00:00'. SQLSTATE: 22009
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1438)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:228)
at org.apache.spark.sql.execution.QueryExecution.simpleString(QueryExecution.scala:277)
at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:324)
at org.apache.spark.sql.execution.QueryExecution.explainString(QueryExecution.scala:292)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:148)
at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:268)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:124)
at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:108)
at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:107)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:124)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:291)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:123)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:77)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:233)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2199)
at org.apache.spark.sql.Dataset.collect(Dataset.scala:1473)
... 30 elided
Caused by: java.time.DateTimeException: Invalid ID for ZoneOffset, invalid format: 1
at java.base/java.time.ZoneOffset.of(ZoneOffset.java:243)
at java.base/java.time.ZoneId.of(ZoneId.java:404)
at java.base/java.time.ZoneId.of(ZoneId.java:358)
at java.base/java.time.ZoneId.of(ZoneId.java:314)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:49)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 50 more
Suppressed: org.apache.spark.util.Utils$OriginalTryStackTraceException: Full stacktrace of original doTryWithCallerStacktrace caller
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError(ExecutionErrors.scala:254)
at org.apache.spark.sql.errors.ExecutionErrors.zoneOffsetError$(ExecutionErrors.scala:246)
at org.apache.spark.sql.errors.ExecutionErrors$.zoneOffsetError(ExecutionErrors.scala:258)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId(SparkDateTimeUtils.scala:52)
at org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.getZoneId$(SparkDateTimeUtils.scala:43)
at org.apache.spark.sql.catalyst.util.DateTimeUtils$.getZoneId(DateTimeUtils.scala:41)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.$anonfun$nullSafeEval$3(datetimeExpressions.scala:2856)
at scala.Option.map(Option.scala:242)
at org.apache.spark.sql.catalyst.expressions.MakeTimestamp.nullSafeEval(datetimeExpressions.scala:2856)
at org.apache.spark.sql.catalyst.expressions.SeptenaryExpression.eval(Expression.scala:1201)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:82)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.$anonfun$constantFolding$4(expressions.scala:102)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1231)
at org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1230)
at org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:573)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.org$apache$spark$sql$catalyst$optimizer$ConstantFolding$$constantFolding(expressions.scala:102)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.$anonfun$applyOrElse$1(expressions.scala:106)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:223)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:235)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:241)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79)
at org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:241)
at org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:312)
at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:246)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:106)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1.applyOrElse(expressions.scala:105)
at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:470)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:105)
at org.apache.spark.sql.catalyst.optimizer.ConstantFolding$.apply(expressions.scala:48)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:242)
at scala.collection.LinearSeqOps.foldLeft(LinearSeq.scala:183)
at scala.collection.LinearSeqOps.foldLeft$(LinearSeq.scala:179)
at scala.collection.immutable.List.foldLeft(List.scala:79)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:239)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:231)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:231)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:201)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:201)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$2(QueryExecution.scala:182)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:258)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:634)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:258)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:706)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:257)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyOptimizedPlan$1(QueryExecution.scala:178)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:192)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyExecutedPlan$1(QueryExecution.scala:214)
at scala.util.Try$.apply(Try.scala:217)
at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
... 50 more
```
### Why are the changes needed?
The suppressed Exception was too verbose.
### Does this PR introduce _any_ user-facing change?
Yes. Stacktraces of Exceptions coming from a LazyTry (e.g. QueryExecution plans) should be more readable.
### How was this patch tested?
Unit tests were adapted.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48882 from juliuszsompolski/SPARK-50338.
Lead-authored-by: Julek Sompolski <Juliusz Sompolski>
Co-authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../main/scala/org/apache/spark/util/Utils.scala | 37 +++--
.../scala/org/apache/spark/util/UtilsSuite.scala | 166 +++++++++++++++++++--
2 files changed, 173 insertions(+), 30 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 9e7ba6d879aa..668657c87b39 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1355,8 +1355,10 @@ private[spark] object Utils
val TRY_WITH_CALLER_STACKTRACE_FULL_STACKTRACE =
"Full stacktrace of original doTryWithCallerStacktrace caller"
- val TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE =
- "Stacktrace under doTryWithCallerStacktrace"
+ class OriginalTryStackTraceException()
+ extends Exception(TRY_WITH_CALLER_STACKTRACE_FULL_STACKTRACE) {
+ var doTryWithCallerStacktraceDepth: Int = 0
+ }
/**
* Use Try with stacktrace substitution for the caller retrieving the error.
@@ -1384,14 +1386,19 @@ private[spark] object Utils
val commonSuffixLen = origStackTrace.reverse.zip(currentStackTrace.reverse).takeWhile {
case (exElem, currentElem) => exElem == currentElem
}.length
- val belowEx = new Exception(TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE)
- belowEx.setStackTrace(origStackTrace.dropRight(commonSuffixLen))
- ex.addSuppressed(belowEx)
-
- // keep the full original stack trace in a suppressed exception.
- val fullEx = new Exception(TRY_WITH_CALLER_STACKTRACE_FULL_STACKTRACE)
- fullEx.setStackTrace(origStackTrace)
- ex.addSuppressed(fullEx)
+ // Add the full stack trace of the original caller as the suppressed exception.
+ // It may already be there if it's a nested call to doTryWithCallerStacktrace.
+ val origEx = ex.getSuppressed.find { e =>
+ e.isInstanceOf[OriginalTryStackTraceException]
+ }.getOrElse {
+ val fullEx = new OriginalTryStackTraceException()
+ fullEx.setStackTrace(origStackTrace)
+ ex.addSuppressed(fullEx)
+ fullEx
+ }.asInstanceOf[OriginalTryStackTraceException]
+ // Update the depth of the stack of the current doTryWithCallerStacktrace, for stitching
+ // it with the stack of getTryWithCallerStacktrace.
+ origEx.doTryWithCallerStacktraceDepth = origStackTrace.size - commonSuffixLen
case Success(_) => // nothing
}
t
@@ -1407,7 +1414,7 @@ private[spark] object Utils
* Full stack trace of the original doTryWithCallerStacktrace caller can be retrieved with
* ```
* ex.getSuppressed.find { e =>
- * e.getMessage == Utils.TRY_WITH_CALLER_STACKTRACE_FULL_STACKTRACE
+ * e.isInstanceOf[Utils.OriginalTryStackTraceException]
* }
* ```
*
@@ -1417,13 +1424,15 @@ private[spark] object Utils
*/
def getTryWithCallerStacktrace[T](t: Try[T]): T = t match {
case Failure(ex) =>
- val belowStacktrace = ex.getSuppressed.find { e =>
+ val originalStacktraceEx = ex.getSuppressed.find { e =>
// added in doTryWithCallerStacktrace
- e.getMessage == TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE
+ e.isInstanceOf[OriginalTryStackTraceException]
}.getOrElse {
// If we don't have the expected stacktrace information, just rethrow
throw ex
- }.getStackTrace
+ }.asInstanceOf[OriginalTryStackTraceException]
+ val belowStacktrace = originalStacktraceEx.getStackTrace
+ .take(originalStacktraceEx.doTryWithCallerStacktraceDepth)
// We are modifying and throwing the original exception. It would be better if we could
// return a copy, but we can't easily clone it and preserve. If this is accessed from
// multiple threads that then look at the stack trace, this could break.
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index a6e3345fc600..febc59e5c536 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1581,26 +1581,14 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties {
// at org.apache.spark.util.UtilsSuite.$anonfun$new$165(UtilsSuite.scala:1658)
// ... 56 more
// scalastyle:on line.size.limit
- val origSt = e1.getSuppressed.find(
- _.getMessage == Utils.TRY_WITH_CALLER_STACKTRACE_FULL_STACKTRACE)
+ val origSt = e1.getSuppressed.find(_.isInstanceOf[Utils.OriginalTryStackTraceException])
assert(origSt.isDefined)
assert(origSt.get.getStackTrace.exists(_.getMethodName == "throwException"))
assert(origSt.get.getStackTrace.exists(_.getMethodName == "callDoTry"))
- // The stack trace under Try should be in the suppressed exceptions.
- // Example:
- // Suppressed: java.lang.Exception: Stacktrace under doTryWithCallerStacktrace
- // at org.apache.spark.util.UtilsSuite.throwException(UtilsSuite.scala: 1640)
- // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTry$1(UtilsSuite.scala: 1645)
- // at scala.util.Try$.apply(Try.scala: 213)
- // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala: 1586)
- val trySt = e1.getSuppressed.find(
- _.getMessage == Utils.TRY_WITH_CALLER_STACKTRACE_TRY_STACKTRACE)
- assert(trySt.isDefined)
- // calls under callDoTry should be present.
- assert(trySt.get.getStackTrace.exists(_.getMethodName == "throwException"))
- // callDoTry should be removed.
- assert(!trySt.get.getStackTrace.exists(_.getMethodName == "callDoTry"))
+ // Should save the depth of the stack trace under doTryWithCallerStacktrace.
+ assert(origSt.get.asInstanceOf[Utils.OriginalTryStackTraceException]
+ .doTryWithCallerStacktraceDepth == 4)
val e2 = intercept[Exception] {
callGetTryAgain(t)
@@ -1633,6 +1621,152 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties {
assert(st1Again.exists(_.getMethodName == "callGetTryAgain"))
assert(!st1Again.exists(_.getMethodName == "callGetTry"))
}
+
+ private def callGetTryFromNested(t: Try[String]): String = {
+ Utils.getTryWithCallerStacktrace(t)
+ }
+
+ private def callDoTryNested(): Try[String] = {
+ Utils.doTryWithCallerStacktrace {
+ val t = callDoTry()
+ val e = intercept[Exception] {
+ callGetTryFromNested(t)
+ }
+
+ // Uncomment for manual inspection
+ //
+ // println("\nIntercepted in callDoTryNested:")
+ // e.printStackTrace()
+ //
+ // scalastyle:off line.size.limit
+ // java.lang.Exception: test
+ // at org.apache.spark.util.UtilsSuite.throwException(UtilsSuite.scala:1529)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTry$1(UtilsSuite.scala:1534)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1438)
+ // ----> at org.apache.spark.util.UtilsSuite.callGetTryFromNested(UtilsSuite.scala:1626) <---- STITCHED.
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNested$2(UtilsSuite.scala:1633)
+ // at org.scalatest.Assertions.intercept(Assertions.scala:749)
+ // at org.scalatest.Assertions.intercept$(Assertions.scala:746)
+ // at org.scalatest.funsuite.AnyFunSuite.intercept(AnyFunSuite.scala:1564)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNested$1(UtilsSuite.scala:1632)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTryNested(UtilsSuite.scala:1630)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNestedNested$1(UtilsSuite.scala:1655)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTryNestedNested(UtilsSuite.scala:1654)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$new$172(UtilsSuite.scala:1674)
+ // ...
+ // Suppressed: org.apache.spark.util.Utils$OriginalTryStackTraceException: Full stacktrace of original doTryWithCallerStacktrace caller
+ // at org.apache.spark.util.UtilsSuite.throwException(UtilsSuite.scala:1529)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTry$1(UtilsSuite.scala:1534)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTry(UtilsSuite.scala:1534)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNested$1(UtilsSuite.scala:1631)
+ // ...
+ // scalastyle:on line.size.limit
+
+ assert(e.getStackTrace.exists(_.getMethodName == "callGetTryFromNested"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTryFromNestedNested"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTry"))
+ assert(e.getSuppressed.length == 1)
+
+ Utils.getTryWithCallerStacktrace(t)
+ }
+ }
+
+ private def callGetTryFromNestedNested(t: Try[String]): String = {
+ Utils.getTryWithCallerStacktrace(t)
+ }
+
+ private def callDoTryNestedNested(): Try[String] = {
+ Utils.doTryWithCallerStacktrace {
+ val t = callDoTryNested()
+ val e = intercept[Exception] {
+ callGetTryFromNestedNested(t)
+ }
+
+ // Uncomment for manual inspection
+ //
+ // println("\nIntercepted in callDoTryNestedNested:")
+ // e.printStackTrace()
+ //
+ // scalastyle:off line.size.limit
+ // java.lang.Exception: test
+ // at org.apache.spark.util.UtilsSuite.throwException(UtilsSuite.scala:1529)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTry$1(UtilsSuite.scala:1534)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTry(UtilsSuite.scala:1534)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNested$1(UtilsSuite.scala:1631)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1438)
+ // ----> at org.apache.spark.util.UtilsSuite.callGetTryFromNestedNested(UtilsSuite.scala:1650) <---- STITCHED.
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNestedNested$2(UtilsSuite.scala:1657)
+ // at org.scalatest.Assertions.intercept(Assertions.scala:749)
+ // at org.scalatest.Assertions.intercept$(Assertions.scala:746)
+ // at org.scalatest.funsuite.AnyFunSuite.intercept(AnyFunSuite.scala:1564)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNestedNested$1(UtilsSuite.scala:1656)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTryNestedNested(UtilsSuite.scala:1654)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$new$172(UtilsSuite.scala:1674)
+ // scalastyle:on line.size.limit
+
+ assert(e.getStackTrace.exists(_.getMethodName == "callGetTryFromNestedNested"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTryFromNested"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTry"))
+ assert(e.getSuppressed.length == 1)
+
+ Utils.getTryWithCallerStacktrace(t)
+ }
+ }
+
+ test("nested doTryWithCallerStacktrace and getTryWithCallerStacktrace") {
+ val t = callDoTryNestedNested()
+
+ val e = intercept[Exception] {
+ callGetTry(t)
+ }
+
+ // Uncomment for manual inspection
+ //
+ // println("\nIntercepted in test:")
+ // e.printStackTrace()
+ //
+ // scalastyle:off line.size.limit
+ // java.lang.Exception: test
+ // at org.apache.spark.util.UtilsSuite.throwException(UtilsSuite.scala:1529)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTry$1(UtilsSuite.scala:1534)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTry(UtilsSuite.scala:1534)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNested$1(UtilsSuite.scala:1631)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.UtilsSuite.callDoTryNested(UtilsSuite.scala:1630)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$callDoTryNestedNested$1(UtilsSuite.scala:1655)
+ // at scala.util.Try$.apply(Try.scala:217)
+ // at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1377)
+ // at org.apache.spark.util.Utils$.getTryWithCallerStacktrace(Utils.scala:1438)
+ // ----> at org.apache.spark.util.UtilsSuite.callGetTry(UtilsSuite.scala:1539) <---- STITCHED.
+ // at org.apache.spark.util.UtilsSuite.$anonfun$new$173(UtilsSuite.scala:1677)
+ // at org.scalatest.Assertions.intercept(Assertions.scala:749)
+ // at org.scalatest.Assertions.intercept$(Assertions.scala:746)
+ // at org.scalatest.funsuite.AnyFunSuite.intercept(AnyFunSuite.scala:1564)
+ // at org.apache.spark.util.UtilsSuite.$anonfun$new$172(UtilsSuite.scala:1676)
+ // scalastyle:on line.size.limit
+
+ assert(e.getStackTrace.exists(_.getMethodName == "callGetTry"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTryFromNested"))
+ assert(!e.getStackTrace.exists(_.getMethodName == "callGetTryFromNestedNested"))
+ assert(e.getSuppressed.length == 1)
+ }
}
private class SimpleExtension
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org