You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Shishir Kharel (JIRA)" <ji...@apache.org> on 2016/08/31 20:55:20 UTC
[jira] [Comment Edited] (SPARK-15570) Pregel functions fail when
run multiple times in the same jvm using sequence of graphs
[ https://issues.apache.org/jira/browse/SPARK-15570?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15453331#comment-15453331 ]
Shishir Kharel edited comment on SPARK-15570 at 8/31/16 8:54 PM:
-----------------------------------------------------------------
It looks like the problem does not exist in Spark 2.0.
was (Author: shishir167):
It looks like the problem does not exist with Spark 2.0.
> Pregel functions fail when run multiple times in the same jvm using sequence of graphs
> --------------------------------------------------------------------------------------
>
> Key: SPARK-15570
> URL: https://issues.apache.org/jira/browse/SPARK-15570
> Project: Spark
> Issue Type: Bug
> Affects Versions: 1.6.0
> Environment: Windows, Unix
> Reporter: Shishir Kharel
> Priority: Minor
>
> I am using sbt's test-only option to test specific test suites. When testing the pregel functions, like connectedComponents, shortestPath in a sqeuence of graphs it passes the first time. When I try running it again, it fails and gives a null pointer exception. The code I am using to test is shown below.
> {code:title=GraphXBugTest.scala|borderStyle=solid}
> before {
> if (ProgramContext.sc == null) {
> Logger.getLogger("org").setLevel(Level.OFF)
> Logger.getLogger("akka").setLevel(Level.OFF)
> val conf = new SparkConf().setAppName("Project").setSparkHome(System.getenv("SPARK_HOME")).setMaster("local[2]")
> val sc = new SparkContext(conf)
> ProgramContext.setContext(sc)
> }
> }
> test("shortestPath"){
> val landmarks = Seq(1, 4).map(_.toLong)
> val shortestPaths = Set(
> (1, Map(1 -> 0, 4 -> 2)), (2, Map(1 -> 1, 4 -> 2)), (3, Map(1 -> 2, 4 -> 1)),
> (4, Map(1 -> 2, 4 -> 0)), (5, Map(1 -> 1, 4 -> 1)), (6, Map(1 -> 3, 4 -> 1)))
> val edgeSeq = Seq((1, 2), (1, 5), (2, 3), (2, 5), (3, 4), (4, 5), (4, 6)).flatMap {
> case e => Seq(e, e.swap)
> }
> val edges = ProgramContext.sc.parallelize(edgeSeq).map { case (v1, v2) => (v1.toLong, v2.toLong) }
> val graph = Graph.fromEdgeTuples(edges, 1)
> val graphs: ParSeq[Graph[Int, Int]] = ParSeq(graph)
> graphs.foreach(x => ShortestPaths.run(x, landmarks))
> }
> }
> {code}
> The error message is shown below.
> [info] - shortestPath *** FAILED ***
> [info] org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 2.0 failed 1 times, most r
> ecent failure: Lost task 0.0 in stage 2.0 (TID 4, localhost): java.lang.NullPointerException
> [info] at org.apache.spark.graphx.lib.ShortestPaths$.incrementMap(ShortestPaths.scala:33)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$.org$apache$spark$graphx$lib$ShortestPaths$$sendMessage$1(Shortes
> tPaths.scala:64)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$$anonfun$run$2.apply(ShortestPaths.scala:69)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$$anonfun$run$2.apply(ShortestPaths.scala:69)
> [info] at org.apache.spark.graphx.impl.GraphImpl.org$apache$spark$graphx$impl$GraphImpl$$sendMsg$1(GraphImpl.scala:19
> 7)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
> [info] at org.apache.spark.graphx.impl.EdgePartition.aggregateMessagesEdgeScan(EdgePartition.scala:409)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:265)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:235)
> [info] at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
> [info] at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:126)
> [info] at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
> [info] at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
> [info] at org.apache.spark.scheduler.Task.run(Task.scala:89)
> [info] at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
> [info] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [info] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [info] at java.lang.Thread.run(Thread.java:745)
> [info]
> [info] Driver stacktrace:
> [info] at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStag
> es(DAGScheduler.scala:1431)
> [info] at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1419)
> [info] at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418)
> [info] at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> [info] at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
> [info] at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)
> [info] at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
> [info] at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
> [info] at scala.Option.foreach(Option.scala:236)
> [info] at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)
> [info] ...
> [info] Cause: java.lang.NullPointerException:
> [info] at org.apache.spark.graphx.lib.ShortestPaths$.incrementMap(ShortestPaths.scala:33)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$.org$apache$spark$graphx$lib$ShortestPaths$$sendMessage$1(Shorte
> stPaths.scala:64)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$$anonfun$run$2.apply(ShortestPaths.scala:69)
> [info] at org.apache.spark.graphx.lib.ShortestPaths$$anonfun$run$2.apply(ShortestPaths.scala:69)
> [info] at org.apache.spark.graphx.impl.GraphImpl.org$apache$spark$graphx$impl$GraphImpl$$sendMsg$1(GraphImpl.scala:1
> 97)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
> [info] at org.apache.spark.graphx.impl.EdgePartition.aggregateMessagesEdgeScan(EdgePartition.scala:409)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:265)
> [info] at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:235)
> [info] ...
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org