You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@spark.apache.org by Guillermo Ortiz <ko...@gmail.com> on 2016/05/09 10:51:45 UTC
java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
I'm trying to execute a job with Spark and Kafka and I'm getting this error.
I know that it's becuase the version are not right, but I have been
checking the jar which I import on the SparkUI spark.yarn.secondary.jars
and they are right and the class exists inside *kafka_2.10-0.8.2.1.jar. *
2016-05-05 15:51:53,617 [Driver] ERROR
org.apache.spark.deploy.yarn.ApplicationMaster - User class threw
exception: java.lang.NoClassDefFoundError:
kafka/api/TopicMetadataRequest
java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
spark-submit --conf spark.metrics.conf=metrics.properties --name
"CentralLog" --master yarn-cluster --class spark.CentralLog --files
/opt/centralLogs/conf/log4j.properties --jars
/opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
--executor-memory 1024m --num-executors 4 --executor-cores 2
--driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
/opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
Why doesn't Spark find the class?
activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
*kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
Re: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
Posted by Guillermo Ortiz <ko...@gmail.com>.
The class is in *kafka_2.10-0.8.2.1.jar* not in
spark-streaming-kafka_2.10-1.5.1.jar
I found the error... so embarrasing. I was executing in cluster-mode and I
only had the jars in the gateway. I guess that Spark chose an NodeManager
to execute a container with the Driver and load the libraries, but the
libraries was just in the Gateway.
Anyway, these jars were copied in HDFS by spark-submit with *--jars
$SPARK_CLASSPATH.*
So my question is: Is it really necessary to all jars in all the "local"
directories of the possible Executor nodes?
2016-05-09 14:49 GMT+02:00 Mich Talebzadeh <mi...@gmail.com>:
> That sounds like specific for Kafka. Check this
> https://www.codatlas.com/github.com/apache/kafka/HEAD/core/src/main/scala/kafka/api/TopicMetadataRequest.scala
>
> I cannot see it in
>
> jar tvf spark-streaming-kafka_2.10-1.5.1.jar|grep TopicMetadataRequest
>
>
>
> Dr Mich Talebzadeh
>
>
>
> LinkedIn * https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw
> <https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw>*
>
>
>
> http://talebzadehmich.wordpress.com
>
>
>
> On 9 May 2016 at 13:35, Guillermo Ortiz <ko...@gmail.com> wrote:
>
>> I was looking the log carefully looking for others error. Anyway the complete Exception is:
>>
>> 2016-05-09 13:20:25,646 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>> java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
>> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>> at produban.spark.CentralLog$.createContext(CentralLog.scala:34)
>> at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:85)
>> at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:84)
>> at scala.Option.getOrElse(Option.scala:120)
>> at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:844)
>> at produban.spark.CentralLog$.main(CentralLog.scala:84)
>> at produban.spark.CentralLog.main(CentralLog.scala)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:606)
>> at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:525)*Caused by: java.lang.ClassNotFoundException: kafka.api.TopicMetadataRequest*
>> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
>> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
>>
>>
>> 2016-05-09 13:12 GMT+02:00 Ted Yu <yu...@gmail.com>:
>>
>>> NoClassDefFoundError is different than saying that it could not be loaded from the classpath.
>>>
>>> From my experience, there should be some other error before this error which would give you better idea.
>>>
>>> You can also check whether another version of kafka is embedded in any of the jars listed below.
>>>
>>> Cheers
>>>
>>>
>>> On Mon, May 9, 2016 at 4:00 AM, Guillermo Ortiz <ko...@gmail.com>
>>> wrote:
>>>
>>>> *jar tvf kafka_2.10-0.8.2.1.jar | grep TopicMetadataRequest *
>>>> 1757 Thu Feb 26 14:30:34 CET 2015
>>>> kafka/api/TopicMetadataRequest$$anonfun$1.class
>>>> 1712 Thu Feb 26 14:30:34 CET 2015
>>>> kafka/api/TopicMetadataRequest$$anonfun$readFrom$1.class
>>>> 1437 Thu Feb 26 14:30:34 CET 2015
>>>> kafka/api/TopicMetadataRequest$$anonfun$sizeInBytes$1.class
>>>> 1435 Thu Feb 26 14:30:34 CET 2015
>>>> kafka/api/TopicMetadataRequest$$anonfun$writeTo$1.class
>>>> 8028 Thu Feb 26 14:30:34 CET 2015
>>>> kafka/api/TopicMetadataRequest$.class
>>>> 10377 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest.class
>>>> 5282 Thu Feb 26 14:30:36 CET 2015
>>>> kafka/javaapi/TopicMetadataRequest.class
>>>> 2135 Thu Feb 26 14:30:38 CET 2015
>>>> kafka/server/KafkaApis$$anonfun$handleTopicMetadataRequest$1.class
>>>>
>>>>
>>>> 2016-05-09 12:51 GMT+02:00 Guillermo Ortiz <ko...@gmail.com>:
>>>>
>>>>> I'm trying to execute a job with Spark and Kafka and I'm getting this
>>>>> error.
>>>>>
>>>>> I know that it's becuase the version are not right, but I have been
>>>>> checking the jar which I import on the SparkUI spark.yarn.secondary.jars
>>>>> and they are right and the class exists inside
>>>>> *kafka_2.10-0.8.2.1.jar. *
>>>>>
>>>>> 2016-05-05 15:51:53,617 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>>>>> java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
>>>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
>>>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
>>>>> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>>>>>
>>>>>
>>>>>
>>>>> spark-submit --conf spark.metrics.conf=metrics.properties --name
>>>>> "CentralLog" --master yarn-cluster --class spark.CentralLog --files
>>>>> /opt/centralLogs/conf/log4j.properties --jars
>>>>> /opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
>>>>> --executor-memory 1024m --num-executors 4 --executor-cores 2
>>>>> --driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
>>>>> /opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
>>>>>
>>>>> Why doesn't Spark find the class?
>>>>>
>>>>>
>>>>> activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
>>>>> *kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
>>>>> ,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
>>>>>
>>>>
>>>>
>>>
>>
>
Re: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
Posted by Mich Talebzadeh <mi...@gmail.com>.
That sounds like specific for Kafka. Check this
https://www.codatlas.com/github.com/apache/kafka/HEAD/core/src/main/scala/kafka/api/TopicMetadataRequest.scala
I cannot see it in
jar tvf spark-streaming-kafka_2.10-1.5.1.jar|grep TopicMetadataRequest
Dr Mich Talebzadeh
LinkedIn * https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw
<https://www.linkedin.com/profile/view?id=AAEAAAAWh2gBxianrbJd6zP6AcPCCdOABUrV8Pw>*
http://talebzadehmich.wordpress.com
On 9 May 2016 at 13:35, Guillermo Ortiz <ko...@gmail.com> wrote:
> I was looking the log carefully looking for others error. Anyway the complete Exception is:
>
> 2016-05-09 13:20:25,646 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
> java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
> at produban.spark.CentralLog$.createContext(CentralLog.scala:34)
> at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:85)
> at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:84)
> at scala.Option.getOrElse(Option.scala:120)
> at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:844)
> at produban.spark.CentralLog$.main(CentralLog.scala:84)
> at produban.spark.CentralLog.main(CentralLog.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:525)*Caused by: java.lang.ClassNotFoundException: kafka.api.TopicMetadataRequest*
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
>
>
> 2016-05-09 13:12 GMT+02:00 Ted Yu <yu...@gmail.com>:
>
>> NoClassDefFoundError is different than saying that it could not be loaded from the classpath.
>>
>> From my experience, there should be some other error before this error which would give you better idea.
>>
>> You can also check whether another version of kafka is embedded in any of the jars listed below.
>>
>> Cheers
>>
>>
>> On Mon, May 9, 2016 at 4:00 AM, Guillermo Ortiz <ko...@gmail.com>
>> wrote:
>>
>>> *jar tvf kafka_2.10-0.8.2.1.jar | grep TopicMetadataRequest *
>>> 1757 Thu Feb 26 14:30:34 CET 2015
>>> kafka/api/TopicMetadataRequest$$anonfun$1.class
>>> 1712 Thu Feb 26 14:30:34 CET 2015
>>> kafka/api/TopicMetadataRequest$$anonfun$readFrom$1.class
>>> 1437 Thu Feb 26 14:30:34 CET 2015
>>> kafka/api/TopicMetadataRequest$$anonfun$sizeInBytes$1.class
>>> 1435 Thu Feb 26 14:30:34 CET 2015
>>> kafka/api/TopicMetadataRequest$$anonfun$writeTo$1.class
>>> 8028 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest$.class
>>> 10377 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest.class
>>> 5282 Thu Feb 26 14:30:36 CET 2015
>>> kafka/javaapi/TopicMetadataRequest.class
>>> 2135 Thu Feb 26 14:30:38 CET 2015
>>> kafka/server/KafkaApis$$anonfun$handleTopicMetadataRequest$1.class
>>>
>>>
>>> 2016-05-09 12:51 GMT+02:00 Guillermo Ortiz <ko...@gmail.com>:
>>>
>>>> I'm trying to execute a job with Spark and Kafka and I'm getting this
>>>> error.
>>>>
>>>> I know that it's becuase the version are not right, but I have been
>>>> checking the jar which I import on the SparkUI spark.yarn.secondary.jars
>>>> and they are right and the class exists inside
>>>> *kafka_2.10-0.8.2.1.jar. *
>>>>
>>>> 2016-05-05 15:51:53,617 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>>>> java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
>>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
>>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
>>>> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>>>>
>>>>
>>>>
>>>> spark-submit --conf spark.metrics.conf=metrics.properties --name
>>>> "CentralLog" --master yarn-cluster --class spark.CentralLog --files
>>>> /opt/centralLogs/conf/log4j.properties --jars
>>>> /opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
>>>> --executor-memory 1024m --num-executors 4 --executor-cores 2
>>>> --driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
>>>> /opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
>>>>
>>>> Why doesn't Spark find the class?
>>>>
>>>>
>>>> activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
>>>> *kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
>>>> ,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
>>>>
>>>
>>>
>>
>
Re: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
Posted by Guillermo Ortiz <ko...@gmail.com>.
I was looking the log carefully looking for others error. Anyway the
complete Exception is:
2016-05-09 13:20:25,646 [Driver] ERROR
org.apache.spark.deploy.yarn.ApplicationMaster - User class threw
exception: java.lang.NoClassDefFoundError:
kafka/api/TopicMetadataRequest
java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
at produban.spark.CentralLog$.createContext(CentralLog.scala:34)
at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:85)
at produban.spark.CentralLog$$anonfun$3.apply(CentralLog.scala:84)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:844)
at produban.spark.CentralLog$.main(CentralLog.scala:84)
at produban.spark.CentralLog.main(CentralLog.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:525)*Caused
by: java.lang.ClassNotFoundException: kafka.api.TopicMetadataRequest*
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
2016-05-09 13:12 GMT+02:00 Ted Yu <yu...@gmail.com>:
> NoClassDefFoundError is different than saying that it could not be loaded from the classpath.
>
> From my experience, there should be some other error before this error which would give you better idea.
>
> You can also check whether another version of kafka is embedded in any of the jars listed below.
>
> Cheers
>
>
> On Mon, May 9, 2016 at 4:00 AM, Guillermo Ortiz <ko...@gmail.com>
> wrote:
>
>> *jar tvf kafka_2.10-0.8.2.1.jar | grep TopicMetadataRequest *
>> 1757 Thu Feb 26 14:30:34 CET 2015
>> kafka/api/TopicMetadataRequest$$anonfun$1.class
>> 1712 Thu Feb 26 14:30:34 CET 2015
>> kafka/api/TopicMetadataRequest$$anonfun$readFrom$1.class
>> 1437 Thu Feb 26 14:30:34 CET 2015
>> kafka/api/TopicMetadataRequest$$anonfun$sizeInBytes$1.class
>> 1435 Thu Feb 26 14:30:34 CET 2015
>> kafka/api/TopicMetadataRequest$$anonfun$writeTo$1.class
>> 8028 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest$.class
>> 10377 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest.class
>> 5282 Thu Feb 26 14:30:36 CET 2015
>> kafka/javaapi/TopicMetadataRequest.class
>> 2135 Thu Feb 26 14:30:38 CET 2015
>> kafka/server/KafkaApis$$anonfun$handleTopicMetadataRequest$1.class
>>
>>
>> 2016-05-09 12:51 GMT+02:00 Guillermo Ortiz <ko...@gmail.com>:
>>
>>> I'm trying to execute a job with Spark and Kafka and I'm getting this
>>> error.
>>>
>>> I know that it's becuase the version are not right, but I have been
>>> checking the jar which I import on the SparkUI spark.yarn.secondary.jars
>>> and they are right and the class exists inside
>>> *kafka_2.10-0.8.2.1.jar. *
>>>
>>> 2016-05-05 15:51:53,617 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>>> java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
>>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
>>> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>>>
>>>
>>>
>>> spark-submit --conf spark.metrics.conf=metrics.properties --name
>>> "CentralLog" --master yarn-cluster --class spark.CentralLog --files
>>> /opt/centralLogs/conf/log4j.properties --jars
>>> /opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
>>> --executor-memory 1024m --num-executors 4 --executor-cores 2
>>> --driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
>>> /opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
>>>
>>> Why doesn't Spark find the class?
>>>
>>>
>>> activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
>>> *kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
>>> ,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
>>>
>>
>>
>
Re: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
Posted by Ted Yu <yu...@gmail.com>.
NoClassDefFoundError is different than saying that it could not be
loaded from the classpath.
From my experience, there should be some other error before this error
which would give you better idea.
You can also check whether another version of kafka is embedded in any
of the jars listed below.
Cheers
On Mon, May 9, 2016 at 4:00 AM, Guillermo Ortiz <ko...@gmail.com>
wrote:
> *jar tvf kafka_2.10-0.8.2.1.jar | grep TopicMetadataRequest *
> 1757 Thu Feb 26 14:30:34 CET 2015
> kafka/api/TopicMetadataRequest$$anonfun$1.class
> 1712 Thu Feb 26 14:30:34 CET 2015
> kafka/api/TopicMetadataRequest$$anonfun$readFrom$1.class
> 1437 Thu Feb 26 14:30:34 CET 2015
> kafka/api/TopicMetadataRequest$$anonfun$sizeInBytes$1.class
> 1435 Thu Feb 26 14:30:34 CET 2015
> kafka/api/TopicMetadataRequest$$anonfun$writeTo$1.class
> 8028 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest$.class
> 10377 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest.class
> 5282 Thu Feb 26 14:30:36 CET 2015
> kafka/javaapi/TopicMetadataRequest.class
> 2135 Thu Feb 26 14:30:38 CET 2015
> kafka/server/KafkaApis$$anonfun$handleTopicMetadataRequest$1.class
>
>
> 2016-05-09 12:51 GMT+02:00 Guillermo Ortiz <ko...@gmail.com>:
>
>> I'm trying to execute a job with Spark and Kafka and I'm getting this
>> error.
>>
>> I know that it's becuase the version are not right, but I have been
>> checking the jar which I import on the SparkUI spark.yarn.secondary.jars
>> and they are right and the class exists inside *kafka_2.10-0.8.2.1.jar. *
>>
>> 2016-05-05 15:51:53,617 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
>> java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
>> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
>> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>>
>>
>>
>> spark-submit --conf spark.metrics.conf=metrics.properties --name
>> "CentralLog" --master yarn-cluster --class spark.CentralLog --files
>> /opt/centralLogs/conf/log4j.properties --jars
>> /opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
>> --executor-memory 1024m --num-executors 4 --executor-cores 2
>> --driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
>> /opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
>>
>> Why doesn't Spark find the class?
>>
>>
>> activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
>> *kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
>> ,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
>>
>
>
Re: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
Posted by Guillermo Ortiz <ko...@gmail.com>.
*jar tvf kafka_2.10-0.8.2.1.jar | grep TopicMetadataRequest *
1757 Thu Feb 26 14:30:34 CET 2015
kafka/api/TopicMetadataRequest$$anonfun$1.class
1712 Thu Feb 26 14:30:34 CET 2015
kafka/api/TopicMetadataRequest$$anonfun$readFrom$1.class
1437 Thu Feb 26 14:30:34 CET 2015
kafka/api/TopicMetadataRequest$$anonfun$sizeInBytes$1.class
1435 Thu Feb 26 14:30:34 CET 2015
kafka/api/TopicMetadataRequest$$anonfun$writeTo$1.class
8028 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest$.class
10377 Thu Feb 26 14:30:34 CET 2015 kafka/api/TopicMetadataRequest.class
5282 Thu Feb 26 14:30:36 CET 2015 kafka/javaapi/TopicMetadataRequest.class
2135 Thu Feb 26 14:30:38 CET 2015
kafka/server/KafkaApis$$anonfun$handleTopicMetadataRequest$1.class
2016-05-09 12:51 GMT+02:00 Guillermo Ortiz <ko...@gmail.com>:
> I'm trying to execute a job with Spark and Kafka and I'm getting this
> error.
>
> I know that it's becuase the version are not right, but I have been
> checking the jar which I import on the SparkUI spark.yarn.secondary.jars
> and they are right and the class exists inside *kafka_2.10-0.8.2.1.jar. *
>
> 2016-05-05 15:51:53,617 [Driver] ERROR org.apache.spark.deploy.yarn.ApplicationMaster - User class threw exception: java.lang.NoClassDefFoundError: kafka/api/TopicMetadataRequest
> java.lang.NoClassDefFoundError: *kafka/api/TopicMetadataRequest*
> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:122)
> at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:112)
> at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:409)
>
>
>
> spark-submit --conf spark.metrics.conf=metrics.properties --name
> "CentralLog" --master yarn-cluster --class spark.CentralLog --files
> /opt/centralLogs/conf/log4j.properties --jars
> /opt/centralLogs/lib/kafka_2.10-0.8.2.1.jar,$SPARK_CLASSPATH
> --executor-memory 1024m --num-executors 4 --executor-cores 2
> --driver-memory 1024m --files /opt/centralLogs/conf/metrics.properties
> /opt/centralLogs/libProject/paas-1.0-SNAPSHOT.jar
>
> Why doesn't Spark find the class?
>
>
> activation-1.1.jar,akka-actor_2.10-2.3.11.jar,akka-remote_2.10-2.3.11.jar,akka-slf4j_2.10-2.3.11.jar,ant-1.6.5.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,arpack_combined_all-0.1.jar,asm-3.1.jar,asm-4.1.jar,asm-commons-3.1.jar,asm-commons-4.1.jar,asm-tree-3.1.jar,automaton-1.11-8.jar,avro-1.7.4.jar,avro-ipc-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-mapred-1.7.7-hadoop2.jar,bonecp-0.8.0.RELEASE.jar,breeze_2.10-0.11.2.jar,breeze-macros_2.10-0.11.2.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,carbonite-1.4.0.jar,cascading-core-2.6.3.jar,cascading-hadoop-2.6.3.jar,cascading-local-2.6.3.jar,chill_2.10-0.5.0.jar,chill-java-0.5.0.jar,clj-stacktrace-0.2.2.jar,clj-time-0.4.1.jar,clojure-1.5.1.jar,clout-1.0.1.jar,commons-cli-1.2.jar,commons-codec-1.9.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.4.1.jar,commons-configuration-1.6.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-exec-1.1.jar,commons-fileupload-1.2.1.jar,commons-httpclient-3.1.jar,commons-io-2.3.jar,commons-lang-2.6.jar,commons-lang3-3.3.2.jar,commons-logging-1.1.3.jar,commons-math3-3.4.1.jar,commons-net-2.2.jar,commons-pool-1.5.4.jar,compojure-1.1.3.jar,compress-lzf-1.0.3.jar,concurrentlinkedhashmap-lru-1.2.jar,config-1.2.1.jar,core-1.1.2.jar,core-3.1.1.jar,core.incubator-0.1.0.jar,curator-client-2.6.0.jar,curator-framework-2.6.0.jar,curator-recipes-2.4.0.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,derby-10.10.2.0.jar,disruptor-2.10.4.jar,dom4j-1.6.1.jar,eigenbase-properties-1.1.5.jar,elasticsearch-hadoop-2.2.1.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,groovy-all-2.1.6.jar,gson-2.2.4.jar,guava-14.0.1.jar,guice-3.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.2.0.jar,hadoop-auth-2.2.0.jar,hadoop-client-2.6.0.jar,hadoop-common-2.6.0.jar,hadoop-hdfs-2.2.0.jar,hadoop-mapreduce-client-app-2.6.0.jar,hadoop-mapreduce-client-common-2.6.0.jar,hadoop-mapreduce-client-core-2.6.0.jar,hadoop-mapreduce-client-jobclient-2.6.0.jar,hadoop-mapreduce-client-shuffle-2.6.0.jar,hadoop-yarn-api-2.6.0.jar,hadoop-yarn-client-2.6.0.jar,hadoop-yarn-common-2.6.0.jar,hadoop-yarn-server-applicationhistoryservice-2.6.0.jar,hadoop-yarn-server-common-2.6.0.jar,hadoop-yarn-server-resourcemanager-2.6.0.jar,hadoop-yarn-server-web-proxy-2.6.0.jar,hiccup-0.3.6.jar,hive-ant-1.2.1.jar,hive-common-1.2.1.jar,hive-exec-1.2.1.jar,hive-exec-1.2.1.spark.jar,hive-metastore-1.2.1.jar,hive-metastore-1.2.1.spark.jar,hive-serde-1.2.1.jar,hive-service-1.2.1.jar,hive-shims-0.20S-1.2.1.jar,hive-shims-0.23-1.2.1.jar,hive-shims-1.2.1.jar,hive-shims-common-1.2.1.jar,hive-shims-scheduler-1.2.1.jar,hppc-0.5.2.jar,hsqldb-1.8.0.10.jar,htrace-core-3.0.4.jar,httpclient-4.3.1.jar,httpcore-4.3.jar,httpmime-4.3.1.jar,ivy-2.4.0.jar,jackson-annotations-2.4.0.jar,jackson-core-2.4.4.jar,jackson-core-asl-1.9.11.jar,jackson-databind-2.4.4.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.11.jar,jackson-module-scala_2.10-2.4.4.jar,jackson-xc-1.8.3.jar,janino-2.7.5.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,JavaEWAH-0.3.2.jar,javax.inject-1.jar,javax.servlet-3.0.0.v201112011016.jar,javolution-5.5.1.jar,jaxb-api-2.2.2.jar,jaxb-core-2.2.7.jar,jaxb-impl-2.2.7.jar,jcl-over-slf4j-1.7.10.jar,jdk.tools-1.7.jar,jdo-api-3.0.1.jar,jersey-client-1.9.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.7.1.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-continuation-8.1.10.v20130312.jar,jetty-deploy-8.1.10.v20130312.jar,jetty-http-8.1.10.v20130312.jar,jetty-io-8.1.10.v20130312.jar,jetty-jmx-8.1.10.v20130312.jar,jetty-security-8.1.10.v20130312.jar,jetty-server-8.1.10.v20130312.jar,jetty-servlet-8.1.10.v20130312.jar,jetty-util-6.1.26.jar,jetty-util-8.1.10.v20130312.jar,jetty-webapp-8.1.10.v20130312.jar,jetty-xml-8.1.10.v20130312.jar,jgrapht-core-0.9.0.jar,jgrapht-jdk1.6-0.8.1.jar,jline-1.0.jar,joda-time-2.2.jar,jodd-core-3.5.2.jar,jpam-1.1.jar,jsch-0.1.42.jar,json-20090211.jar,json4s-ast_2.10-3.2.10.jar,json4s-core_2.10-3.2.10.jar,json4s-jackson_2.10-3.2.10.jar,json4s-native_2.10-3.2.10.jar,json-simple-1.1.jar,jsp-2.1-6.1.14.jar,jsp-api-2.1-6.1.14.jar,jsr305-1.3.9.jar,jta-1.1.jar,jtransforms-2.4.0.jar,jul-to-slf4j-1.7.10.jar,junit-3.8.1.jar,
> *kafka_2.10-0.8.2.1.jar,kafka-clients-0.8.2.1.jar*
> ,kfs-0.3.jar,kryo-2.21.jar,leveldbjni-all-1.8.jar,libfb303-0.9.2.jar,libthrift-0.9.2.jar,log4j-1.2.17.jar,log4j-over-slf4j-1.6.6.jar,logback-classic-1.0.13.jar,logback-core-1.0.13.jar,lucene-analyzers-common-4.10.3.jar,lucene-analyzers-kuromoji-4.10.3.jar,lucene-analyzers-phonetic-4.10.3.jar,lucene-codecs-4.10.3.jar,lucene-core-4.10.3.jar,lucene-expressions-4.10.3.jar,lucene-grouping-4.10.3.jar,lucene-highlighter-4.10.3.jar,lucene-join-4.10.3.jar,lucene-memory-4.10.3.jar,lucene-misc-4.10.3.jar,lucene-queries-4.10.3.jar,lucene-queryparser-4.10.3.jar,lucene-spatial-4.10.3.jar,lucene-suggest-4.10.3.jar,lz4-1.3.0.jar,mail-1.4.1.jar,math.numeric-tower-0.0.1.jar,mesos-0.21.1-shaded-protobuf.jar,metrics-core-2.2.0.jar,metrics-core-3.0.0.jar,metrics-core-3.1.2.jar,metrics-graphite-3.1.2.jar,metrics-json-3.1.2.jar,metrics-jvm-3.1.2.jar,minlog-1.2.jar,netty-3.8.0.Final.jar,netty-all-4.0.29.Final.jar,noggit-0.5.jar,objenesis-1.2.jar,opencsv-2.3.jar,org.restlet-2.1.1.jar,org.restlet.ext.servlet-2.1.1.jar,oro-2.0.8.jar,paranamer-2.6.jar,parquet-column-1.7.0.jar,parquet-common-1.7.0.jar,parquet-encoding-1.7.0.jar,parquet-format-2.3.0-incubating.jar,parquet-generator-1.7.0.jar,parquet-hadoop-1.7.0.jar,parquet-hadoop-bundle-1.6.0.jar,parquet-jackson-1.7.0.jar,pig-0.15.0.jar,pmml-agent-1.1.15.jar,pmml-model-1.1.15.jar,pmml-schema-1.1.15.jar,protobuf-java-2.5.0.jar,py4j-0.8.2.1.jar,pyrolite-4.4.jar,quasiquotes_2.10-2.0.0-M8.jar,reflectasm-1.07-shaded.jar,riffle-0.1-dev.jar,ring-core-1.1.5.jar,ring-devel-0.3.11.jar,ring-jetty-adapter-0.3.11.jar,ring-servlet-0.3.11.jar,RoaringBitmap-0.4.5.jar,scala-compiler-2.10.0.jar,scalactic_2.10-3.0.0-M15.jar,scala-library-2.10.4.jar,scalap-2.10.0.jar,scala-reflect-2.10.5.jar,scalatest_2.10-3.0.0-M15.jar,servlet-api-2.5-6.1.14.jar,servlet-api-2.5.jar,slf4j-api-1.7.6.jar,snakeyaml-1.11.jar,snappy-0.2.jar,snappy-java-1.1.1.7.jar,solr-core-4.10.3.jar,solr-solrj-4.10.3.jar,spark-catalyst_2.10-1.6.1.jar,spark-core_2.10-1.5.0.jar,spark-graphx_2.10-1.5.2.jar,spark-hive_2.10-1.5.2.jar,spark-launcher_2.10-1.5.0.jar,spark-mllib_2.10-1.5.2.jar,spark-network-common_2.10-1.5.0.jar,spark-network-shuffle_2.10-1.5.0.jar,spark-solr-1.1.2.jar,spark-sql_2.10-1.6.1.jar,spark-streaming_2.10-1.5.0.jar,spark-streaming-kafka_2.10-1.5.0.jar,spark-streaming-twitter_2.10-1.5.2.jar,spark-unsafe_2.10-1.5.0.jar,spatial4j-0.4.1.jar,spire_2.10-0.7.4.jar,spire-macros_2.10-0.7.4.jar,ST4-4.0.4.jar,stax-api-1.0.1.jar,stax-api-1.0-2.jar,storm-core-0.9.6.jar,stream-2.7.0.jar,tachyon-client-0.7.1.jar,tachyon-underfs-hdfs-0.7.1.jar,tachyon-underfs-local-0.7.1.jar,tools.cli-0.2.4.jar,tools.logging-0.2.3.jar,tools.macro-0.1.0.jar,twitter4j-core-3.0.3.jar,twitter4j-stream-3.0.3.jar,uncommons-maths-1.2.2a.jar,unused-1.0.0.jar,velocity-1.5.jar,wstx-asl-3.2.7.jar,xmlenc-0.52.jar,xz-1.0.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar
>