You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Vinoth Sankar <vi...@gmail.com> on 2015/10/01 14:22:46 UTC
How to connect HadoopHA from spark
Hi,
How do i connect HadoopHA from SPARK. I tried overwriting hadoop
configurations from sparkCong. But Still I'm getting UnknownHostException
with following trace
java.lang.IllegalArgumentException: java.net.UnknownHostException: ABC at
org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:377)
at
org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:240)
at
org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:144)
at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:579) at
org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:524) at
org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:146)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2397)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:89) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2431) at
org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2413) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:368) at
org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1521) at
org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1528) at
org.apache.spark.deploy.master.Master.rebuildSparkUI(Master.scala:747) at
org.apache.spark.deploy.master.Master.removeApplication(Master.scala:710)
at
org.apache.spark.deploy.master.Master.finishApplication(Master.scala:688)
at
org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
at
org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
at scala.Option.foreach(Option.scala:236) at
org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1.applyOrElse(Master.scala:432)
at
scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33)
at
scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33)
at
scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25)
at
org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:53)
at
org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42)
at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) at
org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42)
at akka.actor.Actor$class.aroundReceive(Actor.scala:465) at
org.apache.spark.deploy.master.Master.aroundReceive(Master.scala:52) at
akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) at
akka.actor.ActorCell.invoke(ActorCell.scala:487) at
akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) at
akka.dispatch.Mailbox.run(Mailbox.scala:220) at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: java.net.UnknownHostException: ABC ... 38 more
Regards
Vinoth Sankar
Re: How to connect HadoopHA from spark
Posted by Ted Yu <yu...@gmail.com>.
Have you setup HADOOP_CONF_DIR in spark-env.sh correctly ?
Cheers
On Thu, Oct 1, 2015 at 5:22 AM, Vinoth Sankar <vi...@gmail.com> wrote:
> Hi,
>
> How do i connect HadoopHA from SPARK. I tried overwriting hadoop
> configurations from sparkCong. But Still I'm getting UnknownHostException
> with following trace
>
> java.lang.IllegalArgumentException: java.net.UnknownHostException: ABC at
> org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:377)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:240)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:144)
> at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:579) at
> org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:524) at
> org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:146)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2397)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:89) at
> org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2431) at
> org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2413) at
> org.apache.hadoop.fs.FileSystem.get(FileSystem.java:368) at
> org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1521) at
> org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1528) at
> org.apache.spark.deploy.master.Master.rebuildSparkUI(Master.scala:747) at
> org.apache.spark.deploy.master.Master.removeApplication(Master.scala:710)
> at
> org.apache.spark.deploy.master.Master.finishApplication(Master.scala:688)
> at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
> at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
> at scala.Option.foreach(Option.scala:236) at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1.applyOrElse(Master.scala:432)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25)
> at
> org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:53)
> at
> org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42)
> at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) at
> org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42)
> at akka.actor.Actor$class.aroundReceive(Actor.scala:465) at
> org.apache.spark.deploy.master.Master.aroundReceive(Master.scala:52) at
> akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) at
> akka.actor.ActorCell.invoke(ActorCell.scala:487) at
> akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) at
> akka.dispatch.Mailbox.run(Mailbox.scala:220) at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
> at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> Caused by: java.net.UnknownHostException: ABC ... 38 more
>
>
> Regards
>
> Vinoth Sankar
>
>
>
Re: How to connect HadoopHA from spark
Posted by Adam McElwee <ad...@mcelwee.me>.
Do you have all of the required HDFS HA config options in your override?
I think these are the minimum required for HA:
dfs.nameservices
dfs.ha.namenodes.{nameservice ID}
dfs.namenode.rpc-address.{nameservice ID}.{name node ID}
On Thu, Oct 1, 2015 at 7:22 AM, Vinoth Sankar <vi...@gmail.com> wrote:
> Hi,
>
> How do i connect HadoopHA from SPARK. I tried overwriting hadoop
> configurations from sparkCong. But Still I'm getting UnknownHostException
> with following trace
>
> java.lang.IllegalArgumentException: java.net.UnknownHostException: ABC at
> org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:377)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:240)
> at
> org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:144)
> at org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:579) at
> org.apache.hadoop.hdfs.DFSClient.(DFSClient.java:524) at
> org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:146)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2397)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:89) at
> org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2431) at
> org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2413) at
> org.apache.hadoop.fs.FileSystem.get(FileSystem.java:368) at
> org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1521) at
> org.apache.spark.util.Utils$.getHadoopFileSystem(Utils.scala:1528) at
> org.apache.spark.deploy.master.Master.rebuildSparkUI(Master.scala:747) at
> org.apache.spark.deploy.master.Master.removeApplication(Master.scala:710)
> at
> org.apache.spark.deploy.master.Master.finishApplication(Master.scala:688)
> at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
> at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$29.apply(Master.scala:432)
> at scala.Option.foreach(Option.scala:236) at
> org.apache.spark.deploy.master.Master$$anonfun$receiveWithLogging$1.applyOrElse(Master.scala:432)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33)
> at
> scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25)
> at
> org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:53)
> at
> org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42)
> at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) at
> org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42)
> at akka.actor.Actor$class.aroundReceive(Actor.scala:465) at
> org.apache.spark.deploy.master.Master.aroundReceive(Master.scala:52) at
> akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) at
> akka.actor.ActorCell.invoke(ActorCell.scala:487) at
> akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) at
> akka.dispatch.Mailbox.run(Mailbox.scala:220) at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
> at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> Caused by: java.net.UnknownHostException: ABC ... 38 more
>
>
> Regards
>
> Vinoth Sankar
>
>
>