You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Nan Zhu <zh...@gmail.com> on 2014/05/03 19:04:25 UTC

Spark-1.0.0-rc3 compiled against Hadoop 2.3.0 cannot read HDFS 2.3.0?

Hi, all 

I compiled the Spark-1.0.0-rc3 against Hadoop 2.3.0 with 

SPARK_HADOOP_VERSION=2.3.0 sbt/sbt assembly

and deploy the spark with HDFS 2.3.0 (yarn = false), it seems that it cannot read data from HDFS

the following exception is thrown

java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$CreateSnapshotRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet; 
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2531)
at java.lang.Class.privateGetPublicMethods(Class.java:2651)
at java.lang.Class.privateGetPublicMethods(Class.java:2661)
at java.lang.Class.getMethods(Class.java:1467)
at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:426)
at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:323)
at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:636)
at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:722)
at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92)
at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537)
at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:334)
at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:241)
at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:141)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:569)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:512)
at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:142)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2316)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:90)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2350)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2332)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:369)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296)
at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:221)
at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:172)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1085)
at org.apache.spark.rdd.RDD.count(RDD.scala:836)
at $iwC$$iwC$$iwC$$iwC.<init>(<console>:15)
at $iwC$$iwC$$iwC.<init>(<console>:20)
at $iwC$$iwC.<init>(<console>:22)
at $iwC.<init>(<console>:24)
at <init>(<console>:26)
at .<init>(<console>:30)
at .<clinit>(<console>)
at .<init>(<console>:7)
at .<clinit>(<console>)
at $print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:788)
at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1056)
at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:614)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:645)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
at org.apache.spark.repl.Main$.main(Main.scala:31)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)



Anyone met the same issue before?

Best,

-- 
Nan Zhu



Re: Spark-1.0.0-rc3 compiled against Hadoop 2.3.0 cannot read HDFS 2.3.0?

Posted by Nan Zhu <zh...@gmail.com>.
Finally, got it 

I have to update my jdk to jdk7 

Best, 

-- 
Nan Zhu


On Saturday, May 3, 2014 at 4:52 PM, Nan Zhu wrote:

> I changed protobuf version to 2.5.0 and use make-distribution to generate the file,  
> 
> everything works (I can access HDFS via spark-shell ) except when I tried to compile my application against the new spark jar file
> 
> it shows
> 
> [error] error while loading <root>, invalid CEN header (bad signature) 
> scala.reflect.internal.MissingRequirementError: object scala.runtime in compiler mirror not found.
> at scala.reflect.internal.MissingRequirementError$.signal(MissingRequirementError.scala:16)
> at scala.reflect.internal.MissingRequirementError$.notFound(MissingRequirementError.scala:17)
> at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:48)
> at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:40)
> at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:61)
> at scala.reflect.internal.Mirrors$RootsBase.getPackage(Mirrors.scala:172)
> at scala.reflect.internal.Mirrors$RootsBase.getRequiredPackage(Mirrors.scala:175)
> at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackage$lzycompute(Definitions.scala:183)
> at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackage(Definitions.scala:183)
> at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackageClass$lzycompute(Definitions.scala:184)
> at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackageClass(Definitions.scala:184)
> at scala.reflect.internal.Definitions$DefinitionsClass.AnnotationDefaultAttr$lzycompute(Definitions.scala:1024)
> at scala.reflect.internal.Definitions$DefinitionsClass.AnnotationDefaultAttr(Definitions.scala:1023)
> at scala.reflect.internal.Definitions$DefinitionsClass.syntheticCoreClasses$lzycompute(Definitions.scala:1153)
> at scala.reflect.internal.Definitions$DefinitionsClass.syntheticCoreClasses(Definitions.scala:1152)
> at scala.reflect.internal.Definitions$DefinitionsClass.symbolsNotPresentInBytecode$lzycompute(Definitions.scala:1196)
> at scala.reflect.internal.Definitions$DefinitionsClass.symbolsNotPresentInBytecode(Definitions.scala:1196)
> at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1261)
> at scala.tools.nsc.Global$Run.<init>(Global.scala:1290)
> at xsbt.CachedCompiler0$$anon$2.<init>(CompilerInterface.scala:113)
> at xsbt.CachedCompiler0.run(CompilerInterface.scala:113)
> at xsbt.CachedCompiler0.run(CompilerInterface.scala:99)
> at xsbt.CompilerInterface.run(CompilerInterface.scala:27)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:622)
> at sbt.compiler.AnalyzingCompiler.call(AnalyzingCompiler.scala:102)
> at sbt.compiler.AnalyzingCompiler.compile(AnalyzingCompiler.scala:48)
> at sbt.compiler.AnalyzingCompiler.compile(AnalyzingCompiler.scala:41)
> at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply$mcV$sp(AggressiveCompile.scala:99)
> at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply(AggressiveCompile.scala:99)
> at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply(AggressiveCompile.scala:99)
> at sbt.compiler.AggressiveCompile.sbt$compiler$AggressiveCompile$$timed(AggressiveCompile.scala:166)
> at sbt.compiler.AggressiveCompile$$anonfun$3.compileScala$1(AggressiveCompile.scala:98)
> at sbt.compiler.AggressiveCompile$$anonfun$3.apply(AggressiveCompile.scala:143)
> at sbt.compiler.AggressiveCompile$$anonfun$3.apply(AggressiveCompile.scala:87)
> at sbt.inc.IncrementalCompile$$anonfun$doCompile$1.apply(Compile.scala:39)
> at sbt.inc.IncrementalCompile$$anonfun$doCompile$1.apply(Compile.scala:37)
> at sbt.inc.IncrementalCommon.cycle(Incremental.scala:99)
> at sbt.inc.Incremental$$anonfun$1.apply(Incremental.scala:38)
> at sbt.inc.Incremental$$anonfun$1.apply(Incremental.scala:37)
> at sbt.inc.Incremental$.manageClassfiles(Incremental.scala:65)
> at sbt.inc.Incremental$.compile(Incremental.scala:37)
> at sbt.inc.IncrementalCompile$.apply(Compile.scala:27)
> at sbt.compiler.AggressiveCompile.compile2(AggressiveCompile.scala:157)
> at sbt.compiler.AggressiveCompile.compile1(AggressiveCompile.scala:71)
> at sbt.compiler.AggressiveCompile.apply(AggressiveCompile.scala:46)
> at sbt.Compiler$.apply(Compiler.scala:75)
> at sbt.Compiler$.apply(Compiler.scala:66)
> at sbt.Defaults$.sbt$Defaults$$compileTaskImpl(Defaults.scala:743)
> at sbt.Defaults$$anonfun$compileTask$1.apply(Defaults.scala:735)
> at sbt.Defaults$$anonfun$compileTask$1.apply(Defaults.scala:735)
> at scala.Function1$$anonfun$compose$1.apply(Function1.scala:47)
> at sbt.$tilde$greater$$anonfun$$u2219$1.apply(TypeFunctions.scala:42)
> at sbt.std.Transform$$anon$4.work(System.scala:64)
> at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
> at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
> at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:18)
> at sbt.Execute.work(Execute.scala:244)
> at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
> at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
> at sbt.ConcurrentRestrictions$$anon$4$$anonfun$1.apply(ConcurrentRestrictions.scala:160)
> at sbt.CompletionService$$anon$2.call(CompletionService.scala:30)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:701)
> [error] (compile:compile) scala.reflect.internal.MissingRequirementError: object scala.runtime in compiler mirror not found.
> [error] Total time: 3 s, completed May 3, 2014 8:40:09 PM
> 
> 
> -- 
> Nan Zhu
> 
> 
> On Saturday, May 3, 2014 at 1:04 PM, Nan Zhu wrote:
> 
> > Hi, all 
> > 
> > I compiled the Spark-1.0.0-rc3 against Hadoop 2.3.0 with 
> > 
> > SPARK_HADOOP_VERSION=2.3.0 sbt/sbt assembly
> > 
> > and deploy the spark with HDFS 2.3.0 (yarn = false), it seems that it cannot read data from HDFS
> > 
> > the following exception is thrown
> > 
> > java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$CreateSnapshotRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet; 
> > at java.lang.ClassLoader.defineClass1(Native Method)
> > at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
> > at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
> > at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
> > at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
> > at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
> > at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> > at java.security.AccessController.doPrivileged(Native Method)
> > at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> > at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
> > at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> > at java.lang.Class.getDeclaredMethods0(Native Method)
> > at java.lang.Class.privateGetDeclaredMethods(Class.java:2531)
> > at java.lang.Class.privateGetPublicMethods(Class.java:2651)
> > at java.lang.Class.privateGetPublicMethods(Class.java:2661)
> > at java.lang.Class.getMethods(Class.java:1467)
> > at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:426)
> > at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:323)
> > at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:636)
> > at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:722)
> > at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92)
> > at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537)
> > at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:334)
> > at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:241)
> > at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:141)
> > at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:569)
> > at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:512)
> > at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:142)
> > at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2316)
> > at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:90)
> > at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2350)
> > at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2332)
> > at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:369)
> > at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296)
> > at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:221)
> > at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
> > at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:172)
> > at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
> > at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
> > at scala.Option.getOrElse(Option.scala:120)
> > at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
> > at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
> > at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
> > at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
> > at scala.Option.getOrElse(Option.scala:120)
> > at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
> > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1085)
> > at org.apache.spark.rdd.RDD.count(RDD.scala:836)
> > at $iwC$$iwC$$iwC$$iwC.<init>(<console>:15)
> > at $iwC$$iwC$$iwC.<init>(<console>:20)
> > at $iwC$$iwC.<init>(<console>:22)
> > at $iwC.<init>(<console>:24)
> > at <init>(<console>:26)
> > at .<init>(<console>:30)
> > at .<clinit>(<console>)
> > at .<init>(<console>:7)
> > at .<clinit>(<console>)
> > at $print(<console>)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:606)
> > at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:788)
> > at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1056)
> > at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:614)
> > at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:645)
> > at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
> > at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
> > at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> > at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
> > at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
> > at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
> > at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
> > at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
> > at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> > at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> > at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
> > at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
> > at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
> > at org.apache.spark.repl.Main$.main(Main.scala:31)
> > at org.apache.spark.repl.Main.main(Main.scala)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:606)
> > at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
> > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
> > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> > 
> > 
> > 
> > Anyone met the same issue before?
> > 
> > Best,
> > 
> > -- 
> > Nan Zhu
> > 
> > 
> 


Re: Spark-1.0.0-rc3 compiled against Hadoop 2.3.0 cannot read HDFS 2.3.0?

Posted by Nan Zhu <zh...@gmail.com>.
I changed protobuf version to 2.5.0 and use make-distribution to generate the file,  

everything works (I can access HDFS via spark-shell ) except when I tried to compile my application against the new spark jar file

it shows

[error] error while loading <root>, invalid CEN header (bad signature) 
scala.reflect.internal.MissingRequirementError: object scala.runtime in compiler mirror not found.
at scala.reflect.internal.MissingRequirementError$.signal(MissingRequirementError.scala:16)
at scala.reflect.internal.MissingRequirementError$.notFound(MissingRequirementError.scala:17)
at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:48)
at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:40)
at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:61)
at scala.reflect.internal.Mirrors$RootsBase.getPackage(Mirrors.scala:172)
at scala.reflect.internal.Mirrors$RootsBase.getRequiredPackage(Mirrors.scala:175)
at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackage$lzycompute(Definitions.scala:183)
at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackage(Definitions.scala:183)
at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackageClass$lzycompute(Definitions.scala:184)
at scala.reflect.internal.Definitions$DefinitionsClass.RuntimePackageClass(Definitions.scala:184)
at scala.reflect.internal.Definitions$DefinitionsClass.AnnotationDefaultAttr$lzycompute(Definitions.scala:1024)
at scala.reflect.internal.Definitions$DefinitionsClass.AnnotationDefaultAttr(Definitions.scala:1023)
at scala.reflect.internal.Definitions$DefinitionsClass.syntheticCoreClasses$lzycompute(Definitions.scala:1153)
at scala.reflect.internal.Definitions$DefinitionsClass.syntheticCoreClasses(Definitions.scala:1152)
at scala.reflect.internal.Definitions$DefinitionsClass.symbolsNotPresentInBytecode$lzycompute(Definitions.scala:1196)
at scala.reflect.internal.Definitions$DefinitionsClass.symbolsNotPresentInBytecode(Definitions.scala:1196)
at scala.reflect.internal.Definitions$DefinitionsClass.init(Definitions.scala:1261)
at scala.tools.nsc.Global$Run.<init>(Global.scala:1290)
at xsbt.CachedCompiler0$$anon$2.<init>(CompilerInterface.scala:113)
at xsbt.CachedCompiler0.run(CompilerInterface.scala:113)
at xsbt.CachedCompiler0.run(CompilerInterface.scala:99)
at xsbt.CompilerInterface.run(CompilerInterface.scala:27)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:622)
at sbt.compiler.AnalyzingCompiler.call(AnalyzingCompiler.scala:102)
at sbt.compiler.AnalyzingCompiler.compile(AnalyzingCompiler.scala:48)
at sbt.compiler.AnalyzingCompiler.compile(AnalyzingCompiler.scala:41)
at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply$mcV$sp(AggressiveCompile.scala:99)
at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply(AggressiveCompile.scala:99)
at sbt.compiler.AggressiveCompile$$anonfun$3$$anonfun$compileScala$1$1.apply(AggressiveCompile.scala:99)
at sbt.compiler.AggressiveCompile.sbt$compiler$AggressiveCompile$$timed(AggressiveCompile.scala:166)
at sbt.compiler.AggressiveCompile$$anonfun$3.compileScala$1(AggressiveCompile.scala:98)
at sbt.compiler.AggressiveCompile$$anonfun$3.apply(AggressiveCompile.scala:143)
at sbt.compiler.AggressiveCompile$$anonfun$3.apply(AggressiveCompile.scala:87)
at sbt.inc.IncrementalCompile$$anonfun$doCompile$1.apply(Compile.scala:39)
at sbt.inc.IncrementalCompile$$anonfun$doCompile$1.apply(Compile.scala:37)
at sbt.inc.IncrementalCommon.cycle(Incremental.scala:99)
at sbt.inc.Incremental$$anonfun$1.apply(Incremental.scala:38)
at sbt.inc.Incremental$$anonfun$1.apply(Incremental.scala:37)
at sbt.inc.Incremental$.manageClassfiles(Incremental.scala:65)
at sbt.inc.Incremental$.compile(Incremental.scala:37)
at sbt.inc.IncrementalCompile$.apply(Compile.scala:27)
at sbt.compiler.AggressiveCompile.compile2(AggressiveCompile.scala:157)
at sbt.compiler.AggressiveCompile.compile1(AggressiveCompile.scala:71)
at sbt.compiler.AggressiveCompile.apply(AggressiveCompile.scala:46)
at sbt.Compiler$.apply(Compiler.scala:75)
at sbt.Compiler$.apply(Compiler.scala:66)
at sbt.Defaults$.sbt$Defaults$$compileTaskImpl(Defaults.scala:743)
at sbt.Defaults$$anonfun$compileTask$1.apply(Defaults.scala:735)
at sbt.Defaults$$anonfun$compileTask$1.apply(Defaults.scala:735)
at scala.Function1$$anonfun$compose$1.apply(Function1.scala:47)
at sbt.$tilde$greater$$anonfun$$u2219$1.apply(TypeFunctions.scala:42)
at sbt.std.Transform$$anon$4.work(System.scala:64)
at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
at sbt.Execute$$anonfun$submit$1$$anonfun$apply$1.apply(Execute.scala:237)
at sbt.ErrorHandling$.wideConvert(ErrorHandling.scala:18)
at sbt.Execute.work(Execute.scala:244)
at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
at sbt.Execute$$anonfun$submit$1.apply(Execute.scala:237)
at sbt.ConcurrentRestrictions$$anon$4$$anonfun$1.apply(ConcurrentRestrictions.scala:160)
at sbt.CompletionService$$anon$2.call(CompletionService.scala:30)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:701)
[error] (compile:compile) scala.reflect.internal.MissingRequirementError: object scala.runtime in compiler mirror not found.
[error] Total time: 3 s, completed May 3, 2014 8:40:09 PM


-- 
Nan Zhu


On Saturday, May 3, 2014 at 1:04 PM, Nan Zhu wrote:

> Hi, all 
> 
> I compiled the Spark-1.0.0-rc3 against Hadoop 2.3.0 with 
> 
> SPARK_HADOOP_VERSION=2.3.0 sbt/sbt assembly
> 
> and deploy the spark with HDFS 2.3.0 (yarn = false), it seems that it cannot read data from HDFS
> 
> the following exception is thrown
> 
> java.lang.VerifyError: class org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$CreateSnapshotRequestProto overrides final method getUnknownFields.()Lcom/google/protobuf/UnknownFieldSet; 
> at java.lang.ClassLoader.defineClass1(Native Method)
> at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
> at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
> at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
> at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> at java.lang.Class.getDeclaredMethods0(Native Method)
> at java.lang.Class.privateGetDeclaredMethods(Class.java:2531)
> at java.lang.Class.privateGetPublicMethods(Class.java:2651)
> at java.lang.Class.privateGetPublicMethods(Class.java:2661)
> at java.lang.Class.getMethods(Class.java:1467)
> at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:426)
> at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:323)
> at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:636)
> at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:722)
> at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:92)
> at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:537)
> at org.apache.hadoop.hdfs.NameNodeProxies.createNNProxyWithClientProtocol(NameNodeProxies.java:334)
> at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:241)
> at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:141)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:569)
> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:512)
> at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:142)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2316)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:90)
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2350)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2332)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:369)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:296)
> at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:221)
> at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:270)
> at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:172)
> at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
> at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
> at scala.Option.getOrElse(Option.scala:120)
> at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
> at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
> at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
> at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
> at scala.Option.getOrElse(Option.scala:120)
> at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1085)
> at org.apache.spark.rdd.RDD.count(RDD.scala:836)
> at $iwC$$iwC$$iwC$$iwC.<init>(<console>:15)
> at $iwC$$iwC$$iwC.<init>(<console>:20)
> at $iwC$$iwC.<init>(<console>:22)
> at $iwC.<init>(<console>:24)
> at <init>(<console>:26)
> at .<init>(<console>:30)
> at .<clinit>(<console>)
> at .<init>(<console>:7)
> at .<clinit>(<console>)
> at $print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:788)
> at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1056)
> at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:614)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:645)
> at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
> at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
> at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
> at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
> at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
> at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
> at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
> at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
> at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
> at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
> at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
> at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
> at org.apache.spark.repl.Main$.main(Main.scala:31)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 
> 
> 
> Anyone met the same issue before?
> 
> Best,
> 
> -- 
> Nan Zhu
> 
>