You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@spark.apache.org by xukun <xu...@huawei.com> on 2014/11/25 15:00:41 UTC
java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds]

submit 12 spark applications in the same time. yarn web page shows: two task fail.

the cmd:
./spark-submit    --class org.apache.spark.examples.JavaWordCount   --master yarn-cluster   ---executor-memory 2g   ../lib/spark-examples_2.10-1.1.0.jar     hdfs://hacluster/bigData

driver log of one fail task:
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
Exception in thread "Driver" java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds]
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
	at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
	at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
	at scala.concurrent.Await$.result(package.scala:107)
	at akka.remote.Remoting.start(Remoting.scala:173)
	at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184)
	at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579)
	at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577)
	at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588)
	at akka.actor.ActorSystem$.apply(ActorSystem.scala:111)
	at akka.actor.ActorSystem$.apply(ActorSystem.scala:104)
	at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121)
	at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54)
	at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53)
	at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458)
	at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
	at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448)
	at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56)
	at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161)
	at org.apache.spark.SparkContext.<init>(SparkContext.scala:213)
	at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
2014-11-23 18:41:19,010 | INFO  | [main] | Registered signal handlers for [TERM, HUP, INT] | org.apache.spark.util.SignalLogger$.register(SignalLogger.scala:47)
2014-11-23 18:41:54,403 | WARN  | [main] | Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | org.apache.hadoop.util.NativeCodeLoader.<clinit>(NativeCodeLoader.java:62)
2014-11-23 18:42:10,319 | INFO  | [main] | ApplicationAttemptId: appattempt_1416732306135_0043_000001 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,213 | INFO  | [main] | Changing view acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,280 | INFO  | [main] | Changing modify acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,300 | INFO  | [main] | SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(omm, spark); users with modify permissions: Set(omm, spark) | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,597 | INFO  | [main] | Starting the user JAR in a separate Thread | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,787 | INFO  | [main] | Waiting for spark context initialization | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,788 | INFO  | [main] | Waiting for spark context initialization ... 0 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:19,801 | WARN  | [Driver] | In Spark 1.0 and later spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone and LOCAL_DIRS in YARN). | org.apache.spark.Logging$class.logWarning(Logging.scala:71)
2014-11-23 18:42:22,495 | INFO  | [Driver] | Changing view acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:22,521 | INFO  | [Driver] | Changing modify acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:22,521 | INFO  | [Driver] | SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(omm, spark); users with modify permissions: Set(omm, spark) | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:28,823 | INFO  | [main] | Waiting for spark context initialization ... 1 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:38,896 | INFO  | [main] | Waiting for spark context initialization ... 2 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:47,737 | INFO  | [sparkDriver-akka.actor.default-dispatcher-3] | Slf4jLogger started | akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.applyOrElse(Slf4jLogger.scala:80)
2014-11-23 18:42:48,938 | INFO  | [main] | Waiting for spark context initialization ... 3 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:53,192 | INFO  | [sparkDriver-akka.actor.default-dispatcher-4] | Starting remoting | akka.event.slf4j.Slf4jLogger$$anonfun$receive$1$$anonfun$applyOrElse$3.apply$mcV$sp(Slf4jLogger.scala:74)
2014-11-23 18:42:58,969 | INFO  | [main] | Waiting for spark context initialization ... 4 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:43:04,721 | ERROR | [sparkDriver-akka.actor.default-dispatcher-3] | Remoting error: [Startup timed out] [
akka.remote.RemoteTransportException: Startup timed out
	at akka.remote.Remoting.akka$remote$Remoting$$notifyError(Remoting.scala:129)
	at akka.remote.Remoting.start(Remoting.scala:191)
	at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184)
	at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579)
	at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577)
	at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588)
	at akka.actor.ActorSystem$.apply(ActorSystem.scala:111)
	at akka.actor.ActorSystem$.apply(ActorSystem.scala:104)
	at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121)
	at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54)
	at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53)
	at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458)
	at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
	at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448)
	at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56)
	at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161)
	at org.apache.spark.SparkContext.<init>(SparkContext.scala:213)
	at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
	at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460)
Caused by: java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds]
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
	at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
	at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
	at scala.concurrent.Await$.result(package.scala:107)
	at akka.remote.Remoting.start(Remoting.scala:173)
	... 22 more



driver log of other fail task:

 2014-11-23 18:49:15,962 | INFO  | [main] | Waiting for spark context initialization ... 9 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,188 | INFO  | [Driver] | Successfully started service 'SparkUI' on port 23702. | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,246 | INFO  | [Driver] | Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,482 | INFO  | [Driver] | Started SparkUI at http://vm-181:23702 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:20,526 | INFO  | [Driver] | Created YarnClusterScheduler | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:26,121 | ERROR | [main] | SparkContext did not initialize after waiting for 100000 ms. Please check earlier log output for errors. Failing the application. | org.apache.spark.Logging$class.logError(Logging.scala:75)
2014-11-23 18:49:27,082 | INFO  | [main] | Final app status: FAILED, exitCode: 13, (reason: Timed out waiting for SparkContext.) | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:27,482 | INFO  | [main] | In securityManager checkExit, exit code: 13 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:28,699 | INFO  | [Thread-3] | Unregistering ApplicationMaster with FAILED (diag message: Timed out waiting for SparkContext.) | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:50:22,133 | INFO  | [Thread-3] | Deleting staging directory .sparkStaging/application_1416732306135_0043 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:50:36,274 | WARN  | [Driver] | interrupted waiting to send rpc request to server | org.apache.hadoop.ipc.Client.call(Client.java:1388)
java.lang.InterruptedException
	at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:400)
	at java.util.concurrent.FutureTask.get(FutureTask.java:187)
	at org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1029)
	at org.apache.hadoop.ipc.Client.call(Client.java:1383)
	at org.apache.hadoop.ipc.Client.call(Client.java:1363)
	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
	at com.sun.proxy.$Proxy16.getFileInfo(Unknown Source)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:701)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
	at com.sun.proxy.$Proxy17.getFileInfo(Unknown Source)
	at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1780)
	at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1165)
	at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1161)
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
	at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1161)
	at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1398)
	at org.apache.spark.util.FileLogger.createLogDir(FileLogger.scala:108)
	at org.apache.spark.util.FileLogger.start(FileLogger.scala:100)
	at org.apache.spark.scheduler.EventLoggingListener.start(EventLoggingListener.scala:74)
	at org.apache.spark.SparkContext.<init>(SparkContext.scala:323)
	at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
	at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460)






---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@spark.apache.org
For additional commands, e-mail: dev-help@spark.apache.org