You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@zeppelin.apache.org by Herman Yu <he...@teeupdata.com> on 2016/09/16 03:32:53 UTC
0.6.1 and spark 2.0.0
With a binary built of zeppelin 0.6.1 and spark 2.0.0 on a HDP 2.4 sandbox, I am getting the following error when spark interpreter (spark conf and zeppelin conf folders are copied from HDP 2.4 and modified accordingly)
I google’d it and it seems related to Yarn timeline server (jersey 1 vs. jersey 2). Has anyone encountered the same issue? Is there a solution/walk-around to make spark 2.0 interpreter work?
Thanks
Herman.
DEBUG [2016-09-15 22:48:08,563] ({pool-2-thread-2} DataTransferSaslUtil.java[getSaslPropertiesResolver]:183) - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection
DEBUG [2016-09-15 22:48:08,566] ({pool-2-thread-2} AbstractService.java[enterState]:452) - Service: org.apache.hadoop.yarn.client.api.impl.YarnClientImpl entered state INITED
ERROR [2016-09-15 22:48:08,590] ({pool-2-thread-2} Utils.java[invokeMethod]:40) -
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkSession(SparkInterpreter.java:343)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkSession(SparkInterpreter.java:216)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:741)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NoClassDefFoundError: com/sun/jersey/api/client/config/ClientConfig
at org.apache.hadoop.yarn.client.api.TimelineClient.createTimelineClient(TimelineClient.java:55)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createTimelineClient(YarnClientImpl.java:181)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:168)
at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:150)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2256)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:831)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:823)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:823)
... 21 more
Caused by: java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 34 more INFO [2016-09-15 22:48:08,592] ({pool-2-thread-2} SparkInterpreter.java[createSparkSession]:344) - Created Spark session
DEBUG [2016-09-15 22:48:08,592] ({pool-1-thread-4} Interpreter.java[getProperty]:179) - key: master, value: yarn-client
ERROR [2016-09-15 22:48:08,593] ({pool-2-thread-2} Job.java[run]:182) - Job failed
java.lang.NullPointerException
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext_2(SparkInterpreter.java:363)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext(SparkInterpreter.java:352)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkContext(SparkInterpreter.java:137)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:743)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
INFO [2016-09-15 22:48:08,598] ({pool-2-thread-2} SchedulerFactory.java[jobFinished]:137) - Job remoteInterpretJob_1473994082381 finished by scheduler org.apache.zeppelin.spark.SparkInterpreter1001065306
Re: 0.6.1 and spark 2.0.0
Posted by Herman Yu <he...@teeupdata.com>.
Thanks Jeff. It worked
> On Sep 16, 2016, at 04:01, Jianfeng (Jeff) Zhang <jz...@hortonworks.com> wrote:
>
>
> Here’s how to disable timeline service.
>
> <property>
> <description>Indicate to clients whether Timeline service is enabled or not.
> If enabled, the TimelineClient library used by end-users will post entities
> and events to the Timeline server.</description>
> <name>yarn.timeline-service.enabled</name>
> <value>false</value>
> </property>
>
>
> Best Regard,
> Jeff Zhang
>
>
> From: Jianfeng Zhang <jzhang@hortonworks.com <ma...@hortonworks.com>>
> Date: Friday, September 16, 2016 at 4:01 PM
> To: "users@zeppelin.apache.org <ma...@zeppelin.apache.org>" <users@zeppelin.apache.org <ma...@zeppelin.apache.org>>
> Subject: Re: 0.6.1 and spark 2.0.0
>
>
> This is a known issue. There’s 2 solutions.
>
> 1. Use the spark 2.0 of HDP 2.5
> 2. Disable timeline service in yarn-site.xml
>
>
> Best Regard,
> Jeff Zhang
>
>
> From: Herman Yu <herman.yu@teeupdata.com <ma...@teeupdata.com>>
> Reply-To: "users@zeppelin.apache.org <ma...@zeppelin.apache.org>" <users@zeppelin.apache.org <ma...@zeppelin.apache.org>>
> Date: Friday, September 16, 2016 at 11:32 AM
> To: "users@zeppelin.apache.org <ma...@zeppelin.apache.org>" <users@zeppelin.apache.org <ma...@zeppelin.apache.org>>
> Subject: 0.6.1 and spark 2.0.0
>
> With a binary built of zeppelin 0.6.1 and spark 2.0.0 on a HDP 2.4 sandbox, I am getting the following error when spark interpreter (spark conf and zeppelin conf folders are copied from HDP 2.4 and modified accordingly)
>
> I google’d it and it seems related to Yarn timeline server (jersey 1 vs. jersey 2). Has anyone encountered the same issue? Is there a solution/walk-around to make spark 2.0 interpreter work?
>
> Thanks
> Herman.
>
>
> DEBUG [2016-09-15 22:48:08,563] ({pool-2-thread-2} DataTransferSaslUtil.java[getSaslPropertiesResolver]:183) - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection
> DEBUG [2016-09-15 22:48:08,566] ({pool-2-thread-2} AbstractService.java[enterState]:452) - Service: org.apache.hadoop.yarn.client.api.impl.YarnClientImpl entered state INITED
> ERROR [2016-09-15 22:48:08,590] ({pool-2-thread-2} Utils.java[invokeMethod]:40) -
> java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
> at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
> at org.apache.zeppelin.spark.SparkInterpreter.createSparkSession(SparkInterpreter.java:343)
> at org.apache.zeppelin.spark.SparkInterpreter.getSparkSession(SparkInterpreter.java:216)
> at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:741)
> at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
> at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
> at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
> at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
> at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.NoClassDefFoundError: com/sun/jersey/api/client/config/ClientConfig
> at org.apache.hadoop.yarn.client.api.TimelineClient.createTimelineClient(TimelineClient.java:55)
> at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createTimelineClient(YarnClientImpl.java:181)
> at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:168)
> at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:150)
> at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
> at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
> at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
> at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2256)
> at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:831)
> at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:823)
> at scala.Option.getOrElse(Option.scala:121)
> at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:823)
> ... 21 more
> Caused by: java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> ... 34 more INFO [2016-09-15 22:48:08,592] ({pool-2-thread-2} SparkInterpreter.java[createSparkSession]:344) - Created Spark session
> DEBUG [2016-09-15 22:48:08,592] ({pool-1-thread-4} Interpreter.java[getProperty]:179) - key: master, value: yarn-client
> ERROR [2016-09-15 22:48:08,593] ({pool-2-thread-2} Job.java[run]:182) - Job failed
> java.lang.NullPointerException
> at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
> at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
> at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext_2(SparkInterpreter.java:363)
> at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext(SparkInterpreter.java:352)
> at org.apache.zeppelin.spark.SparkInterpreter.getSparkContext(SparkInterpreter.java:137)
> at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:743)
> at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
> at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
> at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
> at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
> at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> INFO [2016-09-15 22:48:08,598] ({pool-2-thread-2} SchedulerFactory.java[jobFinished]:137) - Job remoteInterpretJob_1473994082381 finished by scheduler org.apache.zeppelin.spark.SparkInterpreter1001065306
>
>
>
Re: 0.6.1 and spark 2.0.0
Posted by "Jianfeng (Jeff) Zhang" <jz...@hortonworks.com>.
Here’s how to disable timeline service.
<property>
<description>Indicate to clients whether Timeline service is enabled or not.
If enabled, the TimelineClient library used by end-users will post entities
and events to the Timeline server.</description>
<name>yarn.timeline-service.enabled</name>
<value>false</value>
</property>
Best Regard,
Jeff Zhang
From: Jianfeng Zhang <jz...@hortonworks.com>>
Date: Friday, September 16, 2016 at 4:01 PM
To: "users@zeppelin.apache.org<ma...@zeppelin.apache.org>" <us...@zeppelin.apache.org>>
Subject: Re: 0.6.1 and spark 2.0.0
This is a known issue. There’s 2 solutions.
1. Use the spark 2.0 of HDP 2.5
2. Disable timeline service in yarn-site.xml
Best Regard,
Jeff Zhang
From: Herman Yu <he...@teeupdata.com>>
Reply-To: "users@zeppelin.apache.org<ma...@zeppelin.apache.org>" <us...@zeppelin.apache.org>>
Date: Friday, September 16, 2016 at 11:32 AM
To: "users@zeppelin.apache.org<ma...@zeppelin.apache.org>" <us...@zeppelin.apache.org>>
Subject: 0.6.1 and spark 2.0.0
With a binary built of zeppelin 0.6.1 and spark 2.0.0 on a HDP 2.4 sandbox, I am getting the following error when spark interpreter (spark conf and zeppelin conf folders are copied from HDP 2.4 and modified accordingly)
I google’d it and it seems related to Yarn timeline server (jersey 1 vs. jersey 2). Has anyone encountered the same issue? Is there a solution/walk-around to make spark 2.0 interpreter work?
Thanks
Herman.
DEBUG [2016-09-15 22:48:08,563] ({pool-2-thread-2} DataTransferSaslUtil.java[getSaslPropertiesResolver]:183) - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection
DEBUG [2016-09-15 22:48:08,566] ({pool-2-thread-2} AbstractService.java[enterState]:452) - Service: org.apache.hadoop.yarn.client.api.impl.YarnClientImpl entered state INITED
ERROR [2016-09-15 22:48:08,590] ({pool-2-thread-2} Utils.java[invokeMethod]:40) -
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkSession(SparkInterpreter.java:343)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkSession(SparkInterpreter.java:216)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:741)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NoClassDefFoundError: com/sun/jersey/api/client/config/ClientConfig
at org.apache.hadoop.yarn.client.api.TimelineClient.createTimelineClient(TimelineClient.java:55)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createTimelineClient(YarnClientImpl.java:181)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:168)
at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:150)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2256)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:831)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:823)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:823)
... 21 more
Caused by: java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 34 more INFO [2016-09-15 22:48:08,592] ({pool-2-thread-2} SparkInterpreter.java[createSparkSession]:344) - Created Spark session
DEBUG [2016-09-15 22:48:08,592] ({pool-1-thread-4} Interpreter.java[getProperty]:179) - key: master, value: yarn-client
ERROR [2016-09-15 22:48:08,593] ({pool-2-thread-2} Job.java[run]:182) - Job failed
java.lang.NullPointerException
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext_2(SparkInterpreter.java:363)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext(SparkInterpreter.java:352)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkContext(SparkInterpreter.java:137)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:743)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
INFO [2016-09-15 22:48:08,598] ({pool-2-thread-2} SchedulerFactory.java[jobFinished]:137) - Job remoteInterpretJob_1473994082381 finished by scheduler org.apache.zeppelin.spark.SparkInterpreter1001065306
Re: 0.6.1 and spark 2.0.0
Posted by "Jianfeng (Jeff) Zhang" <jz...@hortonworks.com>.
This is a known issue. There’s 2 solutions.
1. Use the spark 2.0 of HDP 2.5
2. Disable timeline service in yarn-site.xml
Best Regard,
Jeff Zhang
From: Herman Yu <he...@teeupdata.com>>
Reply-To: "users@zeppelin.apache.org<ma...@zeppelin.apache.org>" <us...@zeppelin.apache.org>>
Date: Friday, September 16, 2016 at 11:32 AM
To: "users@zeppelin.apache.org<ma...@zeppelin.apache.org>" <us...@zeppelin.apache.org>>
Subject: 0.6.1 and spark 2.0.0
With a binary built of zeppelin 0.6.1 and spark 2.0.0 on a HDP 2.4 sandbox, I am getting the following error when spark interpreter (spark conf and zeppelin conf folders are copied from HDP 2.4 and modified accordingly)
I google’d it and it seems related to Yarn timeline server (jersey 1 vs. jersey 2). Has anyone encountered the same issue? Is there a solution/walk-around to make spark 2.0 interpreter work?
Thanks
Herman.
DEBUG [2016-09-15 22:48:08,563] ({pool-2-thread-2} DataTransferSaslUtil.java[getSaslPropertiesResolver]:183) - DataTransferProtocol not using SaslPropertiesResolver, no QOP found in configuration for dfs.data.transfer.protection
DEBUG [2016-09-15 22:48:08,566] ({pool-2-thread-2} AbstractService.java[enterState]:452) - Service: org.apache.hadoop.yarn.client.api.impl.YarnClientImpl entered state INITED
ERROR [2016-09-15 22:48:08,590] ({pool-2-thread-2} Utils.java[invokeMethod]:40) -
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkSession(SparkInterpreter.java:343)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkSession(SparkInterpreter.java:216)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:741)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NoClassDefFoundError: com/sun/jersey/api/client/config/ClientConfig
at org.apache.hadoop.yarn.client.api.TimelineClient.createTimelineClient(TimelineClient.java:55)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createTimelineClient(YarnClientImpl.java:181)
at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:168)
at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:150)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2256)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:831)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:823)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:823)
... 21 more
Caused by: java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 34 more INFO [2016-09-15 22:48:08,592] ({pool-2-thread-2} SparkInterpreter.java[createSparkSession]:344) - Created Spark session
DEBUG [2016-09-15 22:48:08,592] ({pool-1-thread-4} Interpreter.java[getProperty]:179) - key: master, value: yarn-client
ERROR [2016-09-15 22:48:08,593] ({pool-2-thread-2} Job.java[run]:182) - Job failed
java.lang.NullPointerException
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:33)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext_2(SparkInterpreter.java:363)
at org.apache.zeppelin.spark.SparkInterpreter.createSparkContext(SparkInterpreter.java:352)
at org.apache.zeppelin.spark.SparkInterpreter.getSparkContext(SparkInterpreter.java:137)
at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:743)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:69)
at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:93)
at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:341)
at org.apache.zeppelin.scheduler.Job.run(Job.java:176)
at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
INFO [2016-09-15 22:48:08,598] ({pool-2-thread-2} SchedulerFactory.java[jobFinished]:137) - Job remoteInterpretJob_1473994082381 finished by scheduler org.apache.zeppelin.spark.SparkInterpreter1001065306