You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Jacek Laskowski <ja...@japila.pl> on 2018/03/16 14:56:36 UTC
NPE in Subexpression Elimination optimization
Hi,
I'm working on a minimal test to reproduce the NPE exception that is thrown
in the latest 2.3.0 and earlier 2.2.1 in subexpression elimination
optimization, and am sending it to the mailing list hoping someone notices
something familiar and would shed more light on what might be the root
cause and how to write a test.
I know why Spark throws the NPE technically since there's this @transient
relation: HadoopFsRelation [1] that is not re-created at de-serialization
on executors, but don't know why this @transient is required in the first
place and more importantly how to write a test.
Any hints appreciated.
FYI Disabling subexpression elimination
with spark.sql.subexpressionElimination.enabled Spark configuration
property helps.
[1]
https://github.com/apache/spark/blob/branch-2.3/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala?utf8=%E2%9C%93#L159
Caused by: java.lang.NullPointerException
at
org.apache.spark.sql.execution.FileSourceScanExec.<init>(DataSourceScanExec.scala:167)
at
org.apache.spark.sql.execution.FileSourceScanExec.doCanonicalize(DataSourceScanExec.scala:502)
at
org.apache.spark.sql.execution.FileSourceScanExec.doCanonicalize(DataSourceScanExec.scala:158)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$lzycompute(QueryPlan.scala:210)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(QueryPlan.scala:209)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.doCanonicalize(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$lzycompute(QueryPlan.scala:210)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(QueryPlan.scala:209)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.doCanonicalize(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$lzycompute(QueryPlan.scala:210)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(QueryPlan.scala:209)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:224)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.doCanonicalize(QueryPlan.scala:224)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$lzycompute(QueryPlan.scala:210)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(QueryPlan.scala:209)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.sameResult(QueryPlan.scala:257)
at
org.apache.spark.sql.execution.ScalarSubquery.semanticEquals(subquery.scala:58)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$Expr.equals(EquivalentExpressions.scala:36)
at
scala.collection.mutable.HashTable$class.elemEquals(HashTable.scala:358)
at scala.collection.mutable.HashMap.elemEquals(HashMap.scala:40)
at
scala.collection.mutable.HashTable$class.scala$collection$mutable$HashTable$$findEntry0(HashTable.scala:136)
at scala.collection.mutable.HashTable$class.findEntry(HashTable.scala:132)
at scala.collection.mutable.HashMap.findEntry(HashMap.scala:40)
at scala.collection.mutable.HashMap.get(HashMap.scala:70)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExpr(EquivalentExpressions.scala:54)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:95)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.addExprTree(EquivalentExpressions.scala:96)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext$$anonfun$subexpressionElimination$1.apply(CodeGenerator.scala:1168)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext$$anonfun$subexpressionElimination$1.apply(CodeGenerator.scala:1168)
at scala.collection.immutable.List.foreach(List.scala:381)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext.subexpressionElimination(CodeGenerator.scala:1168)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext.generateExpressions(CodeGenerator.scala:1218)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.createCode(GenerateUnsafeProjection.scala:310)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:373)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.generate(GenerateUnsafeProjection.scala:362)
at
org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:161)
at
org.apache.spark.sql.execution.ProjectExec$$anonfun$9.apply(basicPhysicalOperators.scala:71)
at
org.apache.spark.sql.execution.ProjectExec$$anonfun$9.apply(basicPhysicalOperators.scala:70)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal$1$$anonfun$apply$24.apply(RDD.scala:818)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal$1$$anonfun$apply$24.apply(RDD.scala:818)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:109)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Pozdrawiam,
Jacek Laskowski
----
https://about.me/JacekLaskowski
Mastering Spark SQL https://bit.ly/mastering-spark-sql
Spark Structured Streaming https://bit.ly/spark-structured-streaming
Mastering Kafka Streams https://bit.ly/mastering-kafka-streams
Follow me at https://twitter.com/jaceklaskowski
Re: NPE in Subexpression Elimination optimization
Posted by Jacek Laskowski <ja...@japila.pl>.
Hi,
Filled https://issues.apache.org/jira/browse/SPARK-23731 and am working on
a workaround (aka fix).
Pozdrawiam,
Jacek Laskowski
----
https://about.me/JacekLaskowski
Mastering Spark SQL https://bit.ly/mastering-spark-sql
Spark Structured Streaming https://bit.ly/spark-structured-streaming
Mastering Kafka Streams https://bit.ly/mastering-kafka-streams
Follow me at https://twitter.com/jaceklaskowski
On Fri, Mar 16, 2018 at 3:56 PM, Jacek Laskowski <ja...@japila.pl> wrote:
> Hi,
>
> I'm working on a minimal test to reproduce the NPE exception that is
> thrown in the latest 2.3.0 and earlier 2.2.1 in subexpression elimination
> optimization, and am sending it to the mailing list hoping someone notices
> something familiar and would shed more light on what might be the root
> cause and how to write a test.
>
> I know why Spark throws the NPE technically since there's this @transient
> relation: HadoopFsRelation [1] that is not re-created at de-serialization
> on executors, but don't know why this @transient is required in the first
> place and more importantly how to write a test.
>
> Any hints appreciated.
>
> FYI Disabling subexpression elimination with spark.sql.
> subexpressionElimination.enabled Spark configuration property helps.
>
> [1] https://github.com/apache/spark/blob/branch-2.3/sql/
> core/src/main/scala/org/apache/spark/sql/execution/
> DataSourceScanExec.scala?utf8=%E2%9C%93#L159
>
> Caused by: java.lang.NullPointerException
> at org.apache.spark.sql.execution.FileSourceScanExec.<
> init>(DataSourceScanExec.scala:167)
> at org.apache.spark.sql.execution.FileSourceScanExec.doCanonicalize(
> DataSourceScanExec.scala:502)
> at org.apache.spark.sql.execution.FileSourceScanExec.doCanonicalize(
> DataSourceScanExec.scala:158)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$
> lzycompute(QueryPlan.scala:210)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(
> QueryPlan.scala:209)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.
> doCanonicalize(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$
> lzycompute(QueryPlan.scala:210)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(
> QueryPlan.scala:209)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.
> doCanonicalize(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$
> lzycompute(QueryPlan.scala:210)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(
> QueryPlan.scala:209)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.
> apply(QueryPlan.scala:224)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.TraversableLike$$anonfun$map$
> 1.apply(TraversableLike.scala:234)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> at scala.collection.immutable.List.map(List.scala:285)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.
> doCanonicalize(QueryPlan.scala:224)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized$
> lzycompute(QueryPlan.scala:210)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.canonicalized(
> QueryPlan.scala:209)
> at org.apache.spark.sql.catalyst.plans.QueryPlan.sameResult(
> QueryPlan.scala:257)
> at org.apache.spark.sql.execution.ScalarSubquery.
> semanticEquals(subquery.scala:58)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$Expr.
> equals(EquivalentExpressions.scala:36)
> at scala.collection.mutable.HashTable$class.elemEquals(
> HashTable.scala:358)
> at scala.collection.mutable.HashMap.elemEquals(HashMap.scala:40)
> at scala.collection.mutable.HashTable$class.scala$
> collection$mutable$HashTable$$findEntry0(HashTable.scala:136)
> at scala.collection.mutable.HashTable$class.findEntry(
> HashTable.scala:132)
> at scala.collection.mutable.HashMap.findEntry(HashMap.scala:40)
> at scala.collection.mutable.HashMap.get(HashMap.scala:70)
> at org.apache.spark.sql.catalyst.expressions.
> EquivalentExpressions.addExpr(EquivalentExpressions.scala:54)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:95)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions$$
> anonfun$addExprTree$1.apply(EquivalentExpressions.scala:96)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.EquivalentExpressions.
> addExprTree(EquivalentExpressions.scala:96)
> at org.apache.spark.sql.catalyst.expressions.codegen.
> CodegenContext$$anonfun$subexpressionElimination$1.
> apply(CodeGenerator.scala:1168)
> at org.apache.spark.sql.catalyst.expressions.codegen.
> CodegenContext$$anonfun$subexpressionElimination$1.
> apply(CodeGenerator.scala:1168)
> at scala.collection.immutable.List.foreach(List.scala:381)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext.
> subexpressionElimination(CodeGenerator.scala:1168)
> at org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext.
> generateExpressions(CodeGenerator.scala:1218)
> at org.apache.spark.sql.catalyst.expressions.codegen.
> GenerateUnsafeProjection$.createCode(GenerateUnsafeProjection.scala:310)
> at org.apache.spark.sql.catalyst.expressions.codegen.
> GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:373)
> at org.apache.spark.sql.catalyst.expressions.codegen.
> GenerateUnsafeProjection$.generate(GenerateUnsafeProjection.scala:362)
> at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.
> create(Projection.scala:161)
> at org.apache.spark.sql.execution.ProjectExec$$anonfun$9.apply(
> basicPhysicalOperators.scala:71)
> at org.apache.spark.sql.execution.ProjectExec$$anonfun$9.apply(
> basicPhysicalOperators.scala:70)
> at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal
> $1$$anonfun$apply$24.apply(RDD.scala:818)
> at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal
> $1$$anonfun$apply$24.apply(RDD.scala:818)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(
> MapPartitionsRDD.scala:38)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(
> MapPartitionsRDD.scala:38)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(
> MapPartitionsRDD.scala:38)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(
> MapPartitionsRDD.scala:38)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
> at org.apache.spark.scheduler.Task.run(Task.scala:109)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(
> ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
>
> Pozdrawiam,
> Jacek Laskowski
> ----
> https://about.me/JacekLaskowski
> Mastering Spark SQL https://bit.ly/mastering-spark-sql
> Spark Structured Streaming https://bit.ly/spark-structured-streaming
> Mastering Kafka Streams https://bit.ly/mastering-kafka-streams
> Follow me at https://twitter.com/jaceklaskowski
>