You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Apache Spark (JIRA)" <ji...@apache.org> on 2016/06/21 20:13:58 UTC
[jira] [Commented] (SPARK-16003) SerializationDebugger run into
infinite loop
[ https://issues.apache.org/jira/browse/SPARK-16003?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15342577#comment-15342577 ]
Apache Spark commented on SPARK-16003:
--------------------------------------
User 'ericl' has created a pull request for this issue:
https://github.com/apache/spark/pull/13814
> SerializationDebugger run into infinite loop
> --------------------------------------------
>
> Key: SPARK-16003
> URL: https://issues.apache.org/jira/browse/SPARK-16003
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
> Reporter: Davies Liu
> Priority: Critical
>
> This is observed while debugging https://issues.apache.org/jira/browse/SPARK-15811
> {code}
> sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> java.lang.reflect.Method.invoke(Method.java:497)
> java.io.ObjectStreamClass.invokeWriteReplace(ObjectStreamClass.java:1118)
> sun.reflect.GeneratedMethodAccessor84.invoke(Unknown Source)
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> java.lang.reflect.Method.invoke(Method.java:497) org.apache.spark.serializer.SerializationDebugger$ObjectStreamClassMethods$.invokeWriteReplace$extension(SerializationDebugger.scala:347)
> org.apache.spark.serializer.SerializationDebugger$.org$apache$spark$serializer$SerializationDebugger$$findObjectAndDescriptor(SerializationDebugger.scala:269)
> org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializable(SerializationDebugger.scala:154)
> org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visit(SerializationDebugger.scala:108)
> org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visitSerializable(SerializationDebugger.scala:206)
> org.apache.spark.serializer.SerializationDebugger$SerializationDebugger.visit(SerializationDebugger.scala:108)
> org.apache.spark.serializer.SerializationDebugger$.find(SerializationDebugger.scala:67)
> org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41) org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46) org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
> org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:295)
> org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:288)
> org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:108) org.apache.spark.SparkContext.clean(SparkContext.scala:2038)
> org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndex$1.apply(RDD.scala:789) org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndex$1.apply(RDD.scala:788)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) org.apache.spark.rdd.RDD.withScope(RDD.scala:357) org.apache.spark.rdd.RDD.mapPartitionsWithIndex(RDD.scala:788) org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:355) org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115) org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115) org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133) org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114) org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:240) org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:323) org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:39) org.apache.spark.sql.Dataset$$anonfun$org$apache$spark$sql$Dataset$$execute$1$1.apply(Dataset.scala:2163) org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57) org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2489) org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$execute$1(Dataset.scala:2162) org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collect(Dataset.scala:2169) org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:1905) org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:1904) org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2519) org.apache.spark.sql.Dataset.head(Dataset.scala:1904) org.apache.spark.sql.Dataset.take(Dataset.scala:2119) com.databricks.backend.daemon.driver.OutputAggregator$.withOutputAggregation0(OutputAggregator.scala:80) com.databricks.backend.daemon.driver.OutputAggregator$.withOutputAggregation(OutputAggregator.scala:42) com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$repl$2.apply(ScalaDriverLocal.scala:196) com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$repl$2.apply(ScalaDriverLocal.scala:188) scala.Option.map(Option.scala:145) com.databricks.backend.daemon.driver.ScalaDriverLocal.repl(ScalaDriverLocal.scala:188) com.databricks.backend.daemon.driver.DriverLocal$$anonfun$execute$3.apply(DriverLocal.scala:169) com.databricks.backend.daemon.driver.DriverLocal$$anonfun$execute$3.apply(DriverLocal.scala:169) com.databricks.logging.UsageLogging$$anonfun$withAttributionContext$1.apply(UsageLogging.scala:118) scala.util.DynamicVariable.withValue(DynamicVariable.scala:57) com.databricks.logging.UsageLogging$class.withAttributionContext(UsageLogging.scala:113) com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:31) com.databricks.logging.UsageLogging$class.withAttributionTags(UsageLogging.scala:151) com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:31) com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:168) com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$3.apply(DriverWrapper.scala:483) com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$3.apply(DriverWrapper.scala:483) scala.util.Try$.apply(Try.scala:161) com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:480) com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:381) com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:212) java.lang.Thread.run(Thread.java:745)
> {code}
> We should fix it or disable it by default.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org