You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "cdmikechen (Jira)" <ji...@apache.org> on 2019/12/23 05:41:00 UTC
[jira] [Created] (HUDI-451) Spark on yarn with Hudi get
java.lang.ClassNotFoundException in spark-shell
cdmikechen created HUDI-451:
-------------------------------
Summary: Spark on yarn with Hudi get java.lang.ClassNotFoundException in spark-shell
Key: HUDI-451
URL: https://issues.apache.org/jira/browse/HUDI-451
Project: Apache Hudi (incubating)
Issue Type: Bug
Components: Deployment
Environment: spark2.4.3
hive2.3.3
hadoop2.8.5
Reporter: cdmikechen
Assignee: cdmikechen
Fix For: 0.5.2
When I use this command to start a spark-shell job, spark shell will report *Caused by: java.lang.ClassNotFoundException: org.glassfish.jersey.client.ClientConfig* in my terminal.
{code}
./spark-shell --master yarn --conf spark.sql.hive.convertMetastoreParquet=false --jars /*/*/hudi-spark-bundle-0.5.1-SNAPSHOT.jar
{code}
{code}
java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError: org/glassfish/jersey/client/ClientConfig
at org.glassfish.hk2.utilities.cache.LRUHybridCache.compute(LRUHybridCache.java:315)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl.getAllMethods(ClassReflectionHelperImpl.java:108)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl.getPostConstructMethod(ClassReflectionHelperImpl.java:184)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl.access$200(ClassReflectionHelperImpl.java:56)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl$1.compute(ClassReflectionHelperImpl.java:64)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl$1.compute(ClassReflectionHelperImpl.java:60)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture$1.call(LRUHybridCache.java:115)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture$1.call(LRUHybridCache.java:111)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture.run(LRUHybridCache.java:173)
at org.glassfish.hk2.utilities.cache.LRUHybridCache.compute(LRUHybridCache.java:292)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl.findPostConstruct(ClassReflectionHelperImpl.java:119)
at org.jvnet.hk2.internal.Utilities.findPostConstruct(Utilities.java:1372)
at org.jvnet.hk2.internal.DefaultClassAnalyzer.getPostConstructMethod(DefaultClassAnalyzer.java:130)
at org.glassfish.jersey.internal.inject.JerseyClassAnalyzer.getPostConstructMethod(JerseyClassAnalyzer.java:252)
at org.jvnet.hk2.internal.Utilities.getPostConstruct(Utilities.java:279)
at org.jvnet.hk2.internal.ClazzCreator.initialize(ClazzCreator.java:168)
at org.jvnet.hk2.internal.ClazzCreator.initialize(ClazzCreator.java:179)
at org.jvnet.hk2.internal.SystemDescriptor.internalReify(SystemDescriptor.java:723)
at org.jvnet.hk2.internal.SystemDescriptor.reify(SystemDescriptor.java:678)
at org.jvnet.hk2.internal.ServiceLocatorImpl.reifyDescriptor(ServiceLocatorImpl.java:458)
at org.jvnet.hk2.internal.ServiceLocatorImpl.narrow(ServiceLocatorImpl.java:2205)
at org.jvnet.hk2.internal.ServiceLocatorImpl.access$1200(ServiceLocatorImpl.java:122)
at org.jvnet.hk2.internal.ServiceLocatorImpl$9.compute(ServiceLocatorImpl.java:1350)
at org.jvnet.hk2.internal.ServiceLocatorImpl$9.compute(ServiceLocatorImpl.java:1345)
at org.glassfish.hk2.utilities.cache.internal.WeakCARCacheImpl.compute(WeakCARCacheImpl.java:116)
at org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetAllServiceHandles(ServiceLocatorImpl.java:1407)
at org.jvnet.hk2.internal.ServiceLocatorImpl.getAllServiceHandles(ServiceLocatorImpl.java:1332)
at org.jvnet.hk2.internal.ServiceLocatorImpl.getAllServiceHandles(ServiceLocatorImpl.java:1321)
at org.glassfish.jersey.internal.inject.Providers.getServiceHandles(Providers.java:354)
at org.glassfish.jersey.internal.inject.Providers.getProviders(Providers.java:187)
at org.glassfish.jersey.server.spi.internal.ParameterValueHelper.createValueProviders(ParameterValueHelper.java:130)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider.create(JavaResourceMethodDispatcherProvider.java:78)
at org.glassfish.jersey.server.model.internal.ResourceMethodDispatcherFactory.create(ResourceMethodDispatcherFactory.java:99)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.<init>(ResourceMethodInvoker.java:169)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.<init>(ResourceMethodInvoker.java:102)
at org.glassfish.jersey.server.model.ResourceMethodInvoker$Builder.build(ResourceMethodInvoker.java:153)
at org.glassfish.jersey.server.internal.routing.RuntimeModelBuilder.createInflector(RuntimeModelBuilder.java:128)
at org.glassfish.jersey.server.internal.routing.RuntimeModelBuilder.createMethodRouter(RuntimeModelBuilder.java:115)
at org.glassfish.jersey.server.internal.routing.RuntimeModelBuilder.createResourceMethodRouters(RuntimeModelBuilder.java:309)
at org.glassfish.jersey.server.internal.routing.RuntimeModelBuilder.buildModel(RuntimeModelBuilder.java:198)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder.buildRouting(RuntimeLocatorModelBuilder.java:238)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder.createRouting(RuntimeLocatorModelBuilder.java:221)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder.access$100(RuntimeLocatorModelBuilder.java:77)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder$1.load(RuntimeLocatorModelBuilder.java:153)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder$1.load(RuntimeLocatorModelBuilder.java:150)
at jersey.repackaged.com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3527)
at jersey.repackaged.com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2319)
at jersey.repackaged.com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2282)
at jersey.repackaged.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2197)
at jersey.repackaged.com.google.common.cache.LocalCache.get(LocalCache.java:3937)
at jersey.repackaged.com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3941)
at jersey.repackaged.com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4824)
at org.glassfish.jersey.server.internal.routing.RuntimeLocatorModelBuilder.getRouting(RuntimeLocatorModelBuilder.java:176)
at org.glassfish.jersey.server.internal.routing.SubResourceLocatorRouter.apply(SubResourceLocatorRouter.java:137)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:109)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:92)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:61)
at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:318)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:473)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:427)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:388)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:341)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:228)
at org.spark_project.jetty.servlet.ServletHolder.handle(ServletHolder.java:848)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1772)
at org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.doFilter(AmIpFilter.java:164)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.spark_project.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.spark_project.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.spark_project.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.spark_project.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.spark_project.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.spark_project.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
at org.spark_project.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.spark_project.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.spark_project.jetty.server.Server.handle(Server.java:539)
at org.spark_project.jetty.server.HttpChannel.handle(HttpChannel.java:333)
at org.spark_project.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.spark_project.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.spark_project.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.spark_project.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.spark_project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.spark_project.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError: org/glassfish/jersey/client/ClientConfig
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture.get(LRUHybridCache.java:164)
at org.glassfish.hk2.utilities.cache.LRUHybridCache.compute(LRUHybridCache.java:303)
... 100 more
Caused by: java.lang.NoClassDefFoundError: org/glassfish/jersey/client/ClientConfig
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2701)
at java.lang.Class.getDeclaredMethods(Class.java:1975)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperUtilities$3.run(ClassReflectionHelperUtilities.java:108)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperUtilities$3.run(ClassReflectionHelperUtilities.java:104)
at java.security.AccessController.doPrivileged(Native Method)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperUtilities.secureGetDeclaredMethods(ClassReflectionHelperUtilities.java:104)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperUtilities.getDeclaredMethodWrappers(ClassReflectionHelperUtilities.java:133)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperUtilities.getAllMethodWrappers(ClassReflectionHelperUtilities.java:192)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl$3.compute(ClassReflectionHelperImpl.java:84)
at org.glassfish.hk2.utilities.reflection.internal.ClassReflectionHelperImpl$3.compute(ClassReflectionHelperImpl.java:80)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture$1.call(LRUHybridCache.java:115)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture$1.call(LRUHybridCache.java:111)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.glassfish.hk2.utilities.cache.LRUHybridCache$OriginThreadAwareFuture.run(LRUHybridCache.java:173)
at org.glassfish.hk2.utilities.cache.LRUHybridCache.compute(LRUHybridCache.java:292)
... 100 more
Caused by: java.lang.ClassNotFoundException: org.glassfish.jersey.client.ClientConfig
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 116 more
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)