You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by ShanxT <ma...@gmail.com> on 2014/07/17 19:16:58 UTC
Error while running example/scala application using spark-submit
Hi,
I am receiving below error while submitting any spark example or scala
application. Really appreciate any help.
spark version = 1.0.0
hadoop version = 2.4.0
Windows/Standalone mode
14/07/17 22:13:19 INFO TaskSchedulerImpl: Cancelling stage 0
Exception in thread "main" org.apache.spark.SparkException: Job aborted due
to stage failure: Task 0.0:0 failed 4 times, most recent failure: Exception
failure in TID 6 o
n host java.lang.NullPointerException
java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
org.apache.hadoop.util.Shell.run(Shell.java:418)
org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873)
org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853)
org.apache.spark.util.Utils$.fetchFile(Utils.scala:421)
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:332)
org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:330)
scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
org.apache.spark.executor.Executor.org$apache$spark$executor$Executor$$updateDependencies(Executor.scala:330)
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:168)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1033)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1017)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1015)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at
scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1015)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
at scala.Option.foreach(Option.scala:236)
at
org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:633)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1207)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
at akka.actor.ActorCell.invoke(ActorCell.scala:456)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
at akka.dispatch.Mailbox.run(Mailbox.scala:219)
at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
at
scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at
scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Exception in thread "delete Spark temp dir
C:\Users\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f"
java.io.IOException: Failed to delete: C:\User
s\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f\jars\spark-examples-1.0.0-hadoop2.4.0.jar
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:599)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
at
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
at
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:275)
14/07/17 22:13:20 INFO TaskSchedulerImpl: Stage 0 was cancelled
--
View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/Error-while-running-example-scala-application-using-spark-submit-tp10056.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
Re: Error while running example/scala application using spark-submit
Posted by Sean Owen <so...@cloudera.com>.
I am probably the wrong person to ask as I never use Hadoop on Windows. But
from looking at the code just now it is clearly trying to accommodate
Windows shell commands. Yes I would not be surprised if it still needs
Cygwin.
A slightly broader point is that ideally it doesnt matter whether Hadoop
works on your platform if using Spark locally without Hadoop. I don't know
how feasible it is to separate but there may be some tweaks to avoid
initializing Hadoop in more cases. See the JIRA.
On Jul 17, 2014 7:52 PM, "Stephen Boesch" <ja...@gmail.com> wrote:
> Hi Sean
> RE: Windows and hadoop 2.4.x
>
> HortonWorks - all the hype aside - only supports Windows Server 2008/2012.
> So this general concept of "supporting Windows" is bunk.
>
> Given that - and since the vast majority of Windows users do not happen to
> have Windows Server on their laptop - do you have any further insight into
> what it means to say that hadoop 2.4.x "supports Windows" ? Are you
> referring to cygwin support?
>
>
>
> 2014-07-17 11:13 GMT-07:00 Sean Owen <so...@cloudera.com>:
>
>> I imagine the issue is ultimately combination of Windows and (stock?)
>> Apache Hadoop. I know that in the past, operations like 'chmod' didn't
>> work on Windows since it assumed the existence of POSIX binaries. That
>> should be long since patched up for 2.4.x but there may be a gotcha
>> here that others can comment on.
>>
>> Do I understand that you're trying to run entirely locally, without
>> Hadoop at all?
>> Then I think this sounds like
>> https://issues.apache.org/jira/browse/SPARK-2356 which does deserve
>> attention. The Hadoop APIs get tickled even when they're not used, and
>> this can cause some initialization gotchas on Windows in particular.
>>
>> On Thu, Jul 17, 2014 at 6:16 PM, ShanxT <ma...@gmail.com> wrote:
>> > Hi,
>> >
>> > I am receiving below error while submitting any spark example or scala
>> > application. Really appreciate any help.
>> >
>> > spark version = 1.0.0
>> > hadoop version = 2.4.0
>> > Windows/Standalone mode
>> >
>> > 14/07/17 22:13:19 INFO TaskSchedulerImpl: Cancelling stage 0
>> > Exception in thread "main" org.apache.spark.SparkException: Job aborted
>> due
>> > to stage failure: Task 0.0:0 failed 4 times, most recent failure:
>> Exception
>> > failure in TID 6 o
>> > n host java.lang.NullPointerException
>> > java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
>> > org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
>> > org.apache.hadoop.util.Shell.run(Shell.java:418)
>> >
>> >
>> org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
>> > org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873)
>> > org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853)
>> > org.apache.spark.util.Utils$.fetchFile(Utils.scala:421)
>> >
>> >
>> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:332)
>> >
>> >
>> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:330)
>> >
>> >
>> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
>> >
>> >
>> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
>> >
>> >
>> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
>> >
>> >
>> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
>> > scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
>> > scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
>> >
>> >
>> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
>> >
>> > org.apache.spark.executor.Executor.org
>> $apache$spark$executor$Executor$$updateDependencies(Executor.scala:330)
>> >
>> > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:168)
>> >
>> >
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> >
>> >
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> > java.lang.Thread.run(Thread.java:745)
>> > Driver stacktrace:
>> > at
>> > org.apache.spark.scheduler.DAGScheduler.org
>> $apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1033)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1017)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1015)
>> > at
>> >
>> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
>> > at
>> > scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1015)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
>> > at scala.Option.foreach(Option.scala:236)
>> > at
>> >
>> org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:633)
>> > at
>> >
>> org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1207)
>> > at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
>> > at akka.actor.ActorCell.invoke(ActorCell.scala:456)
>> > at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
>> > at akka.dispatch.Mailbox.run(Mailbox.scala:219)
>> > at
>> >
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
>> > at
>> > scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
>> > at
>> >
>> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
>> > at
>> > scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> > at
>> >
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>> > Exception in thread "delete Spark temp dir
>> >
>> C:\Users\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f"
>> > java.io.IOException: Failed to delete: C:\User
>> >
>> s\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f\jars\spark-examples-1.0.0-hadoop2.4.0.jar
>> > at
>> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:599)
>> > at
>> >
>> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
>> > at
>> >
>> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
>> > at
>> >
>> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
>> > at
>> > scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
>> > at
>> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
>> > at
>> >
>> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
>> > at
>> >
>> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
>> > at
>> >
>> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
>> > at
>> > scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
>> > at
>> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
>> > at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:275)
>> > 14/07/17 22:13:20 INFO TaskSchedulerImpl: Stage 0 was cancelled
>> >
>> >
>> >
>> >
>> > --
>> > View this message in context:
>> http://apache-spark-user-list.1001560.n3.nabble.com/Error-while-running-example-scala-application-using-spark-submit-tp10056.html
>> > Sent from the Apache Spark User List mailing list archive at Nabble.com.
>>
>
>
Re: Error while running example/scala application using spark-submit
Posted by Stephen Boesch <ja...@gmail.com>.
Hi Sean
RE: Windows and hadoop 2.4.x
HortonWorks - all the hype aside - only supports Windows Server 2008/2012.
So this general concept of "supporting Windows" is bunk.
Given that - and since the vast majority of Windows users do not happen to
have Windows Server on their laptop - do you have any further insight into
what it means to say that hadoop 2.4.x "supports Windows" ? Are you
referring to cygwin support?
2014-07-17 11:13 GMT-07:00 Sean Owen <so...@cloudera.com>:
> I imagine the issue is ultimately combination of Windows and (stock?)
> Apache Hadoop. I know that in the past, operations like 'chmod' didn't
> work on Windows since it assumed the existence of POSIX binaries. That
> should be long since patched up for 2.4.x but there may be a gotcha
> here that others can comment on.
>
> Do I understand that you're trying to run entirely locally, without
> Hadoop at all?
> Then I think this sounds like
> https://issues.apache.org/jira/browse/SPARK-2356 which does deserve
> attention. The Hadoop APIs get tickled even when they're not used, and
> this can cause some initialization gotchas on Windows in particular.
>
> On Thu, Jul 17, 2014 at 6:16 PM, ShanxT <ma...@gmail.com> wrote:
> > Hi,
> >
> > I am receiving below error while submitting any spark example or scala
> > application. Really appreciate any help.
> >
> > spark version = 1.0.0
> > hadoop version = 2.4.0
> > Windows/Standalone mode
> >
> > 14/07/17 22:13:19 INFO TaskSchedulerImpl: Cancelling stage 0
> > Exception in thread "main" org.apache.spark.SparkException: Job aborted
> due
> > to stage failure: Task 0.0:0 failed 4 times, most recent failure:
> Exception
> > failure in TID 6 o
> > n host java.lang.NullPointerException
> > java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
> > org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
> > org.apache.hadoop.util.Shell.run(Shell.java:418)
> >
> > org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
> > org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873)
> > org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853)
> > org.apache.spark.util.Utils$.fetchFile(Utils.scala:421)
> >
> >
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:332)
> >
> >
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:330)
> >
> >
> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
> >
> >
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> >
> >
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> >
> >
> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
> > scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
> > scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
> >
> >
> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
> >
> > org.apache.spark.executor.Executor.org
> $apache$spark$executor$Executor$$updateDependencies(Executor.scala:330)
> >
> > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:168)
> >
> >
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> >
> >
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> > java.lang.Thread.run(Thread.java:745)
> > Driver stacktrace:
> > at
> > org.apache.spark.scheduler.DAGScheduler.org
> $apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1033)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1017)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1015)
> > at
> >
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> > at
> > scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1015)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
> > at scala.Option.foreach(Option.scala:236)
> > at
> >
> org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:633)
> > at
> >
> org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1207)
> > at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
> > at akka.actor.ActorCell.invoke(ActorCell.scala:456)
> > at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
> > at akka.dispatch.Mailbox.run(Mailbox.scala:219)
> > at
> >
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
> > at
> > scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
> > at
> >
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> > at
> > scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> > at
> >
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> > Exception in thread "delete Spark temp dir
> >
> C:\Users\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f"
> > java.io.IOException: Failed to delete: C:\User
> >
> s\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f\jars\spark-examples-1.0.0-hadoop2.4.0.jar
> > at
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:599)
> > at
> >
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
> > at
> >
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
> > at
> >
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> > at
> > scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> > at
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
> > at
> >
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
> > at
> >
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
> > at
> >
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> > at
> > scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> > at
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
> > at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:275)
> > 14/07/17 22:13:20 INFO TaskSchedulerImpl: Stage 0 was cancelled
> >
> >
> >
> >
> > --
> > View this message in context:
> http://apache-spark-user-list.1001560.n3.nabble.com/Error-while-running-example-scala-application-using-spark-submit-tp10056.html
> > Sent from the Apache Spark User List mailing list archive at Nabble.com.
>
Re: Error while running example/scala application using
spark-submit
Posted by ShanxT <ma...@gmail.com>.
Thanks Sean,
1) Yes, I am trying to run locally without Hadoop.
2) I also see the error in the provided link while launching spark-shell but
post launch I am able to execute same code I have in the sample application.
Read any local file and perform some reduction operation. But not through
submit command.
--
View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/Error-while-running-example-scala-application-using-spark-submit-tp10056p10064.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
Re: Error while running example/scala application using spark-submit
Posted by Sean Owen <so...@cloudera.com>.
I imagine the issue is ultimately combination of Windows and (stock?)
Apache Hadoop. I know that in the past, operations like 'chmod' didn't
work on Windows since it assumed the existence of POSIX binaries. That
should be long since patched up for 2.4.x but there may be a gotcha
here that others can comment on.
Do I understand that you're trying to run entirely locally, without
Hadoop at all?
Then I think this sounds like
https://issues.apache.org/jira/browse/SPARK-2356 which does deserve
attention. The Hadoop APIs get tickled even when they're not used, and
this can cause some initialization gotchas on Windows in particular.
On Thu, Jul 17, 2014 at 6:16 PM, ShanxT <ma...@gmail.com> wrote:
> Hi,
>
> I am receiving below error while submitting any spark example or scala
> application. Really appreciate any help.
>
> spark version = 1.0.0
> hadoop version = 2.4.0
> Windows/Standalone mode
>
> 14/07/17 22:13:19 INFO TaskSchedulerImpl: Cancelling stage 0
> Exception in thread "main" org.apache.spark.SparkException: Job aborted due
> to stage failure: Task 0.0:0 failed 4 times, most recent failure: Exception
> failure in TID 6 o
> n host java.lang.NullPointerException
> java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
> org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
> org.apache.hadoop.util.Shell.run(Shell.java:418)
>
> org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
> org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873)
> org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853)
> org.apache.spark.util.Utils$.fetchFile(Utils.scala:421)
>
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:332)
>
> org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor$Executor$$updateDependencies$6.apply(Executor.scala:330)
>
> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
>
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
>
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
>
> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
> scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
> scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
>
> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
>
> org.apache.spark.executor.Executor.org$apache$spark$executor$Executor$$updateDependencies(Executor.scala:330)
>
> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:168)
>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> java.lang.Thread.run(Thread.java:745)
> Driver stacktrace:
> at
> org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1033)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1017)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1015)
> at
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> at
> scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
> at
> org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1015)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:633)
> at scala.Option.foreach(Option.scala:236)
> at
> org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:633)
> at
> org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1207)
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
> at akka.actor.ActorCell.invoke(ActorCell.scala:456)
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
> at akka.dispatch.Mailbox.run(Mailbox.scala:219)
> at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
> at
> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
> at
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> Exception in thread "delete Spark temp dir
> C:\Users\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f"
> java.io.IOException: Failed to delete: C:\User
> s\~1\AppData\Local\Temp\spark-88e26679-5a8f-4a37-bf02-41f4b2b46d8f\jars\spark-examples-1.0.0-hadoop2.4.0.jar
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:599)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
> at
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> at
> scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:593)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:592)
> at
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> at
> scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:592)
> at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:275)
> 14/07/17 22:13:20 INFO TaskSchedulerImpl: Stage 0 was cancelled
>
>
>
>
> --
> View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/Error-while-running-example-scala-application-using-spark-submit-tp10056.html
> Sent from the Apache Spark User List mailing list archive at Nabble.com.