You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Madhukar Thota <ma...@gmail.com> on 2016/01/06 05:54:42 UTC
flink kafka scala error
Hi
I am seeing the following error when i am trying to run the jar in Flink
Cluster. I am not sure what dependency is missing.
/opt/DataHUB/flink-0.10.1/bin/flink run datahub-heka-1.0-SNAPSHOT.jar
flink.properties
java.lang.NoClassDefFoundError: scala/collection/GenTraversableOnce$class
at kafka.utils.Pool.<init>(Pool.scala:28)
at
kafka.consumer.FetchRequestAndResponseStatsRegistry$.<init>(FetchRequestAndResponseStats.scala:60)
at
kafka.consumer.FetchRequestAndResponseStatsRegistry$.<clinit>(FetchRequestAndResponseStats.scala)
at kafka.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:39)
at
kafka.javaapi.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:34)
at
org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691)
at
org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281)
at
org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49)
at com.lmig.datahub.heka.Main.main(Main.java:39)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497)
at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395)
at
org.apache.flink.client.program.Client.runBlocking(Client.java:252)
at
org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676)
at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326)
at
org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978)
at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028)
Caused by: java.lang.ClassNotFoundException:
scala.collection.GenTraversableOnce$class
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 20 more
The exception above occurred while trying to run your command.
Here is my pom.xml:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.datahub</groupId>
<artifactId>datahub-heka</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>0.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>0.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>0.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.8.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka</artifactId>
<version>0.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-elasticsearch</artifactId>
<version>0.10.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-shield</artifactId>
<version>1.3.3</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-license-plugin</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.4</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.4</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>elasticsearch-releases</id>
<url>http://maven.elasticsearch.org/releases</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<build>
<plugins>
<!-- We use the maven-shade plugin to create a fat jar
that contains all dependencies
except flink and it's transitive dependencies. The
resulting fat-jar can be executed
on a cluster. Change the value of Program-Class if your
program entry point changes. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<!-- Run shade goal on package phase -->
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<!-- This list contains all
dependencies of flink-dist
Everything else will be packaged
into the fat-jar
-->
<exclude>org.apache.flink:flink-shaded-*</exclude>
<exclude>org.apache.flink:flink-core</exclude>
<exclude>org.apache.flink:flink-java</exclude>
<exclude>org.apache.flink:flink-scala</exclude>
<exclude>org.apache.flink:flink-runtime</exclude>
<exclude>org.apache.flink:flink-optimizer</exclude>
<exclude>org.apache.flink:flink-clients</exclude>
<exclude>org.apache.flink:flink-avro</exclude>
<exclude>org.apache.flink:flink-java-examples</exclude>
<exclude>org.apache.flink:flink-scala-examples</exclude>
<exclude>org.apache.flink:flink-streaming-examples</exclude>
<exclude>org.apache.flink:flink-streaming-java</exclude>
<!-- Also exclude very big
transitive dependencies of Flink
WARNING: You have to remove these
excludes if your code relies on other
versions of these dependencies.
-->
<exclude>org.scala-lang:scala-library</exclude>
<exclude>org.scala-lang:scala-compiler</exclude>
<exclude>org.scala-lang:scala-reflect</exclude>
<exclude>com.amazonaws:aws-java-sdk</exclude>
<exclude>com.typesafe.akka:akka-actor_*</exclude>
<exclude>com.typesafe.akka:akka-remote_*</exclude>
<exclude>com.typesafe.akka:akka-slf4j_*</exclude>
<exclude>io.netty:netty-all</exclude>
<exclude>io.netty:netty</exclude>
<exclude>org.eclipse.jetty:jetty-server</exclude>
<exclude>org.eclipse.jetty:jetty-continuation</exclude>
<exclude>org.eclipse.jetty:jetty-http</exclude>
<exclude>org.eclipse.jetty:jetty-io</exclude>
<exclude>org.eclipse.jetty:jetty-util</exclude>
<exclude>org.eclipse.jetty:jetty-security</exclude>
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>commons-fileupload:commons-fileupload</exclude>
<exclude>org.apache.avro:avro</exclude>
<exclude>commons-collections:commons-collections</exclude>
<exclude>org.codehaus.jackson:jackson-core-asl</exclude>
<exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
<exclude>com.thoughtworks.paranamer:paranamer</exclude>
<exclude>org.xerial.snappy:snappy-java</exclude>
<exclude>org.apache.commons:commons-compress</exclude>
<exclude>org.tukaani:xz</exclude>
<exclude>com.esotericsoftware.kryo:kryo</exclude>
<exclude>com.esotericsoftware.minlog:minlog</exclude>
<exclude>org.objenesis:objenesis</exclude>
<exclude>com.twitter:chill_*</exclude>
<exclude>com.twitter:chill-java</exclude>
<exclude>com.twitter:chill-avro_*</exclude>
<exclude>com.twitter:chill-bijection_*</exclude>
<exclude>com.twitter:bijection-core_*</exclude>
<exclude>com.twitter:bijection-avro_*</exclude>
<exclude>commons-lang:commons-lang</exclude>
<exclude>junit:junit</exclude>
<exclude>de.javakaffee:kryo-serializers</exclude>
<exclude>joda-time:joda-time</exclude>
<exclude>org.apache.commons:commons-lang3</exclude>
<exclude>org.slf4j:slf4j-api</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<exclude>log4j:log4j</exclude>
<exclude>org.apache.commons:commons-math</exclude>
<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
<exclude>commons-logging:commons-logging</exclude>
<exclude>org.apache.httpcomponents:httpclient</exclude>
<exclude>org.apache.httpcomponents:httpcore</exclude>
<exclude>commons-codec:commons-codec</exclude>
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
<exclude>org.codehaus.jettison:jettison</exclude>
<exclude>stax:stax-api</exclude>
<exclude>com.typesafe:config</exclude>
<exclude>org.uncommons.maths:uncommons-maths</exclude>
<exclude>com.github.scopt:scopt_*</exclude>
<exclude>org.mortbay.jetty:servlet-api</exclude>
<exclude>commons-io:commons-io</exclude>
<exclude>commons-cli:commons-cli</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>org.apache.flink:*</artifact>
<excludes>
<exclude>org/apache/flink/shaded/**</exclude>
<exclude>web-docs/**</exclude>
</excludes>
</filter>
<filter>
<!-- Do not copy the signatures in
the META-INF folder.
Otherwise, this might cause
SecurityExceptions when using the JAR. -->
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<!-- add Main-Class to manifest file -->
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.datahub.heka.Main</mainClass>
</transformer>
</transformers>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
</execution>
</executions>
</plugin>
<!-- Configure the jar plugin to add the main class as a
manifest entry -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<configuration>
<archive>
<manifestEntries>
<Main-Class>com.datahub.heka.Main</Main-Class>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.8</source> <!-- If you want to use Java
8, change this to "1.8" -->
<target>1.8</target> <!-- If you want to use Java
8, change this to "1.8" -->
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<!-- A profile that does everyting correctly:
We set the Flink dependencies to provided -->
<id>build-jar</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>0.10.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>0.10.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>0.10.1</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
Any help is appreciated.
Thanks
Re: flink kafka scala error
Posted by Madhukar Thota <ma...@gmail.com>.
I did the solve problem by changing the scala version for Kafka library as
i download the scala_2.11 version of flink (
flink-0.10.1-bin-hadoop27-scala_2.11.tg
<http://www.carfab.com/apachesoftware/flink/flink-0.10.1/flink-0.10.1-bin-hadoop27-scala_2.11.tgz>
z).
Before:
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.8.2.2</version>
</dependency>
After:
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>0.8.2.2</version>
</dependency>
On Wed, Jan 6, 2016 at 4:13 AM, Till Rohrmann <tr...@apache.org> wrote:
> Hi Madhukar,
>
> could you check whether your Flink installation contains the
> flink-dist-0.10.1.jar in the lib folder? This file contains the necessary
> scala-library.jar which you are missing. You can also remove the line
> <exclude>org.scala-lang:scala-library</exclude> which excludes the
> scala-library dependency to be included in the fat jar of your job.
>
> Cheers,
> Till
>
>
> On Wed, Jan 6, 2016 at 5:54 AM, Madhukar Thota <ma...@gmail.com>
> wrote:
>
>> Hi
>>
>> I am seeing the following error when i am trying to run the jar in Flink
>> Cluster. I am not sure what dependency is missing.
>>
>> /opt/DataHUB/flink-0.10.1/bin/flink run datahub-heka-1.0-SNAPSHOT.jar
>> flink.properties
>> java.lang.NoClassDefFoundError: scala/collection/GenTraversableOnce$class
>> at kafka.utils.Pool.<init>(Pool.scala:28)
>> at
>> kafka.consumer.FetchRequestAndResponseStatsRegistry$.<init>(FetchRequestAndResponseStats.scala:60)
>> at
>> kafka.consumer.FetchRequestAndResponseStatsRegistry$.<clinit>(FetchRequestAndResponseStats.scala)
>> at kafka.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:39)
>> at
>> kafka.javaapi.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:34)
>> at
>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691)
>> at
>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281)
>> at
>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49)
>> at com.lmig.datahub.heka.Main.main(Main.java:39)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> at
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:497)
>> at
>> org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497)
>> at
>> org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395)
>> at
>> org.apache.flink.client.program.Client.runBlocking(Client.java:252)
>> at
>> org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676)
>> at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326)
>> at
>> org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978)
>> at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028)
>> Caused by: java.lang.ClassNotFoundException:
>> scala.collection.GenTraversableOnce$class
>> at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
>> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
>> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
>> ... 20 more
>>
>> The exception above occurred while trying to run your command.
>>
>>
>> Here is my pom.xml:
>>
>> <?xml version="1.0" encoding="UTF-8"?>
>> <project xmlns="http://maven.apache.org/POM/4.0.0"
>> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
>> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
>> <modelVersion>4.0.0</modelVersion>
>>
>> <groupId>com.datahub</groupId>
>> <artifactId>datahub-heka</artifactId>
>> <version>1.0-SNAPSHOT</version>
>> <dependencies>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-java</artifactId>
>> <version>0.10.1</version>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-streaming-java</artifactId>
>> <version>0.10.1</version>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-clients</artifactId>
>> <version>0.10.1</version>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.kafka</groupId>
>> <artifactId>kafka_2.10</artifactId>
>> <version>0.8.2.2</version>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-connector-kafka</artifactId>
>> <version>0.10.1</version>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-connector-elasticsearch</artifactId>
>> <version>0.10.1</version>
>> </dependency>
>> <dependency>
>> <groupId>org.elasticsearch</groupId>
>> <artifactId>elasticsearch</artifactId>
>> <version>1.7.2</version>
>> </dependency>
>> <dependency>
>> <groupId>org.elasticsearch</groupId>
>> <artifactId>elasticsearch-shield</artifactId>
>> <version>1.3.3</version>
>> </dependency>
>> <dependency>
>> <groupId>org.elasticsearch</groupId>
>> <artifactId>elasticsearch-license-plugin</artifactId>
>> <version>1.0.0</version>
>> </dependency>
>> <dependency>
>> <groupId>com.fasterxml.jackson.core</groupId>
>> <artifactId>jackson-core</artifactId>
>> <version>2.6.4</version>
>> </dependency>
>> <dependency>
>> <groupId>com.fasterxml.jackson.core</groupId>
>> <artifactId>jackson-databind</artifactId>
>> <version>2.6.4</version>
>> </dependency>
>> </dependencies>
>> <repositories>
>> <repository>
>> <id>elasticsearch-releases</id>
>> <url>http://maven.elasticsearch.org/releases</url>
>> <releases>
>> <enabled>true</enabled>
>> </releases>
>> <snapshots>
>> <enabled>false</enabled>
>> </snapshots>
>> </repository>
>> </repositories>
>> <build>
>> <plugins>
>> <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
>> except flink and it's transitive dependencies. The resulting fat-jar can be executed
>> on a cluster. Change the value of Program-Class if your program entry point changes. -->
>> <plugin>
>> <groupId>org.apache.maven.plugins</groupId>
>> <artifactId>maven-shade-plugin</artifactId>
>> <version>2.4.1</version>
>> <executions>
>> <!-- Run shade goal on package phase -->
>> <execution>
>> <phase>package</phase>
>> <goals>
>> <goal>shade</goal>
>> </goals>
>> <configuration>
>> <artifactSet>
>> <excludes>
>> <!-- This list contains all dependencies of flink-dist
>> Everything else will be packaged into the fat-jar
>> -->
>> <exclude>org.apache.flink:flink-shaded-*</exclude>
>> <exclude>org.apache.flink:flink-core</exclude>
>> <exclude>org.apache.flink:flink-java</exclude>
>> <exclude>org.apache.flink:flink-scala</exclude>
>> <exclude>org.apache.flink:flink-runtime</exclude>
>> <exclude>org.apache.flink:flink-optimizer</exclude>
>> <exclude>org.apache.flink:flink-clients</exclude>
>> <exclude>org.apache.flink:flink-avro</exclude>
>> <exclude>org.apache.flink:flink-java-examples</exclude>
>> <exclude>org.apache.flink:flink-scala-examples</exclude>
>> <exclude>org.apache.flink:flink-streaming-examples</exclude>
>> <exclude>org.apache.flink:flink-streaming-java</exclude>
>>
>> <!-- Also exclude very big transitive dependencies of Flink
>> WARNING: You have to remove these excludes if your code relies on other
>> versions of these dependencies.
>> -->
>> <exclude>org.scala-lang:scala-library</exclude>
>> <exclude>org.scala-lang:scala-compiler</exclude>
>> <exclude>org.scala-lang:scala-reflect</exclude>
>> <exclude>com.amazonaws:aws-java-sdk</exclude>
>> <exclude>com.typesafe.akka:akka-actor_*</exclude>
>> <exclude>com.typesafe.akka:akka-remote_*</exclude>
>> <exclude>com.typesafe.akka:akka-slf4j_*</exclude>
>> <exclude>io.netty:netty-all</exclude>
>> <exclude>io.netty:netty</exclude>
>> <exclude>org.eclipse.jetty:jetty-server</exclude>
>> <exclude>org.eclipse.jetty:jetty-continuation</exclude>
>> <exclude>org.eclipse.jetty:jetty-http</exclude>
>> <exclude>org.eclipse.jetty:jetty-io</exclude>
>> <exclude>org.eclipse.jetty:jetty-util</exclude>
>> <exclude>org.eclipse.jetty:jetty-security</exclude>
>> <exclude>org.eclipse.jetty:jetty-servlet</exclude>
>> <exclude>commons-fileupload:commons-fileupload</exclude>
>> <exclude>org.apache.avro:avro</exclude>
>> <exclude>commons-collections:commons-collections</exclude>
>> <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
>> <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
>> <exclude>com.thoughtworks.paranamer:paranamer</exclude>
>> <exclude>org.xerial.snappy:snappy-java</exclude>
>> <exclude>org.apache.commons:commons-compress</exclude>
>> <exclude>org.tukaani:xz</exclude>
>> <exclude>com.esotericsoftware.kryo:kryo</exclude>
>> <exclude>com.esotericsoftware.minlog:minlog</exclude>
>> <exclude>org.objenesis:objenesis</exclude>
>> <exclude>com.twitter:chill_*</exclude>
>> <exclude>com.twitter:chill-java</exclude>
>> <exclude>com.twitter:chill-avro_*</exclude>
>> <exclude>com.twitter:chill-bijection_*</exclude>
>> <exclude>com.twitter:bijection-core_*</exclude>
>> <exclude>com.twitter:bijection-avro_*</exclude>
>> <exclude>commons-lang:commons-lang</exclude>
>> <exclude>junit:junit</exclude>
>> <exclude>de.javakaffee:kryo-serializers</exclude>
>> <exclude>joda-time:joda-time</exclude>
>> <exclude>org.apache.commons:commons-lang3</exclude>
>> <exclude>org.slf4j:slf4j-api</exclude>
>> <exclude>org.slf4j:slf4j-log4j12</exclude>
>> <exclude>log4j:log4j</exclude>
>> <exclude>org.apache.commons:commons-math</exclude>
>> <exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
>> <exclude>commons-logging:commons-logging</exclude>
>> <exclude>org.apache.httpcomponents:httpclient</exclude>
>> <exclude>org.apache.httpcomponents:httpcore</exclude>
>> <exclude>commons-codec:commons-codec</exclude>
>> <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
>> <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
>> <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
>> <exclude>org.codehaus.jettison:jettison</exclude>
>> <exclude>stax:stax-api</exclude>
>> <exclude>com.typesafe:config</exclude>
>> <exclude>org.uncommons.maths:uncommons-maths</exclude>
>> <exclude>com.github.scopt:scopt_*</exclude>
>> <exclude>org.mortbay.jetty:servlet-api</exclude>
>> <exclude>commons-io:commons-io</exclude>
>> <exclude>commons-cli:commons-cli</exclude>
>> </excludes>
>> </artifactSet>
>> <filters>
>> <filter>
>> <artifact>org.apache.flink:*</artifact>
>> <excludes>
>> <exclude>org/apache/flink/shaded/**</exclude>
>> <exclude>web-docs/**</exclude>
>> </excludes>
>> </filter>
>> <filter>
>> <!-- Do not copy the signatures in the META-INF folder.
>> Otherwise, this might cause SecurityExceptions when using the JAR. -->
>> <artifact>*:*</artifact>
>> <excludes>
>> <exclude>META-INF/*.SF</exclude>
>> <exclude>META-INF/*.DSA</exclude>
>> <exclude>META-INF/*.RSA</exclude>
>> </excludes>
>> </filter>
>> </filters>
>> <transformers>
>> <!-- add Main-Class to manifest file -->
>> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
>> <mainClass>com.datahub.heka.Main</mainClass>
>> </transformer>
>> </transformers>
>> <createDependencyReducedPom>false</createDependencyReducedPom>
>> </configuration>
>> </execution>
>> </executions>
>> </plugin>
>>
>> <!-- Configure the jar plugin to add the main class as a manifest entry -->
>> <plugin>
>> <groupId>org.apache.maven.plugins</groupId>
>> <artifactId>maven-jar-plugin</artifactId>
>> <version>2.5</version>
>> <configuration>
>> <archive>
>> <manifestEntries>
>> <Main-Class>com.datahub.heka.Main</Main-Class>
>> </manifestEntries>
>> </archive>
>> </configuration>
>> </plugin>
>>
>> <plugin>
>> <groupId>org.apache.maven.plugins</groupId>
>> <artifactId>maven-compiler-plugin</artifactId>
>> <version>3.1</version>
>> <configuration>
>> <source>1.8</source> <!-- If you want to use Java 8, change this to "1.8" -->
>> <target>1.8</target> <!-- If you want to use Java 8, change this to "1.8" -->
>> </configuration>
>> </plugin>
>> </plugins>
>>
>>
>> </build>
>> <profiles>
>> <profile>
>> <!-- A profile that does everyting correctly:
>> We set the Flink dependencies to provided -->
>> <id>build-jar</id>
>> <activation>
>> <activeByDefault>false</activeByDefault>
>> </activation>
>> <dependencies>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-java</artifactId>
>> <version>0.10.1</version>
>> <scope>provided</scope>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-streaming-java</artifactId>
>> <version>0.10.1</version>
>> <scope>provided</scope>
>> </dependency>
>> <dependency>
>> <groupId>org.apache.flink</groupId>
>> <artifactId>flink-clients</artifactId>
>> <version>0.10.1</version>
>> <scope>provided</scope>
>> </dependency>
>> </dependencies>
>> </profile>
>> </profiles>
>> </project>
>>
>>
>> Any help is appreciated.
>>
>>
>> Thanks
>>
>>
>
Re: flink kafka scala error
Posted by Till Rohrmann <tr...@apache.org>.
Hi Madhukar,
could you check whether your Flink installation contains the
flink-dist-0.10.1.jar in the lib folder? This file contains the necessary
scala-library.jar which you are missing. You can also remove the line
<exclude>org.scala-lang:scala-library</exclude> which excludes the
scala-library dependency to be included in the fat jar of your job.
Cheers,
Till
On Wed, Jan 6, 2016 at 5:54 AM, Madhukar Thota <ma...@gmail.com>
wrote:
> Hi
>
> I am seeing the following error when i am trying to run the jar in Flink
> Cluster. I am not sure what dependency is missing.
>
> /opt/DataHUB/flink-0.10.1/bin/flink run datahub-heka-1.0-SNAPSHOT.jar
> flink.properties
> java.lang.NoClassDefFoundError: scala/collection/GenTraversableOnce$class
> at kafka.utils.Pool.<init>(Pool.scala:28)
> at
> kafka.consumer.FetchRequestAndResponseStatsRegistry$.<init>(FetchRequestAndResponseStats.scala:60)
> at
> kafka.consumer.FetchRequestAndResponseStatsRegistry$.<clinit>(FetchRequestAndResponseStats.scala)
> at kafka.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:39)
> at
> kafka.javaapi.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:34)
> at
> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691)
> at
> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281)
> at
> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49)
> at com.lmig.datahub.heka.Main.main(Main.java:39)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497)
> at
> org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395)
> at
> org.apache.flink.client.program.Client.runBlocking(Client.java:252)
> at
> org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676)
> at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326)
> at
> org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978)
> at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028)
> Caused by: java.lang.ClassNotFoundException:
> scala.collection.GenTraversableOnce$class
> at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
> ... 20 more
>
> The exception above occurred while trying to run your command.
>
>
> Here is my pom.xml:
>
> <?xml version="1.0" encoding="UTF-8"?>
> <project xmlns="http://maven.apache.org/POM/4.0.0"
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
> <modelVersion>4.0.0</modelVersion>
>
> <groupId>com.datahub</groupId>
> <artifactId>datahub-heka</artifactId>
> <version>1.0-SNAPSHOT</version>
> <dependencies>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-java</artifactId>
> <version>0.10.1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-streaming-java</artifactId>
> <version>0.10.1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-clients</artifactId>
> <version>0.10.1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.kafka</groupId>
> <artifactId>kafka_2.10</artifactId>
> <version>0.8.2.2</version>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-connector-kafka</artifactId>
> <version>0.10.1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-connector-elasticsearch</artifactId>
> <version>0.10.1</version>
> </dependency>
> <dependency>
> <groupId>org.elasticsearch</groupId>
> <artifactId>elasticsearch</artifactId>
> <version>1.7.2</version>
> </dependency>
> <dependency>
> <groupId>org.elasticsearch</groupId>
> <artifactId>elasticsearch-shield</artifactId>
> <version>1.3.3</version>
> </dependency>
> <dependency>
> <groupId>org.elasticsearch</groupId>
> <artifactId>elasticsearch-license-plugin</artifactId>
> <version>1.0.0</version>
> </dependency>
> <dependency>
> <groupId>com.fasterxml.jackson.core</groupId>
> <artifactId>jackson-core</artifactId>
> <version>2.6.4</version>
> </dependency>
> <dependency>
> <groupId>com.fasterxml.jackson.core</groupId>
> <artifactId>jackson-databind</artifactId>
> <version>2.6.4</version>
> </dependency>
> </dependencies>
> <repositories>
> <repository>
> <id>elasticsearch-releases</id>
> <url>http://maven.elasticsearch.org/releases</url>
> <releases>
> <enabled>true</enabled>
> </releases>
> <snapshots>
> <enabled>false</enabled>
> </snapshots>
> </repository>
> </repositories>
> <build>
> <plugins>
> <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
> except flink and it's transitive dependencies. The resulting fat-jar can be executed
> on a cluster. Change the value of Program-Class if your program entry point changes. -->
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-shade-plugin</artifactId>
> <version>2.4.1</version>
> <executions>
> <!-- Run shade goal on package phase -->
> <execution>
> <phase>package</phase>
> <goals>
> <goal>shade</goal>
> </goals>
> <configuration>
> <artifactSet>
> <excludes>
> <!-- This list contains all dependencies of flink-dist
> Everything else will be packaged into the fat-jar
> -->
> <exclude>org.apache.flink:flink-shaded-*</exclude>
> <exclude>org.apache.flink:flink-core</exclude>
> <exclude>org.apache.flink:flink-java</exclude>
> <exclude>org.apache.flink:flink-scala</exclude>
> <exclude>org.apache.flink:flink-runtime</exclude>
> <exclude>org.apache.flink:flink-optimizer</exclude>
> <exclude>org.apache.flink:flink-clients</exclude>
> <exclude>org.apache.flink:flink-avro</exclude>
> <exclude>org.apache.flink:flink-java-examples</exclude>
> <exclude>org.apache.flink:flink-scala-examples</exclude>
> <exclude>org.apache.flink:flink-streaming-examples</exclude>
> <exclude>org.apache.flink:flink-streaming-java</exclude>
>
> <!-- Also exclude very big transitive dependencies of Flink
> WARNING: You have to remove these excludes if your code relies on other
> versions of these dependencies.
> -->
> <exclude>org.scala-lang:scala-library</exclude>
> <exclude>org.scala-lang:scala-compiler</exclude>
> <exclude>org.scala-lang:scala-reflect</exclude>
> <exclude>com.amazonaws:aws-java-sdk</exclude>
> <exclude>com.typesafe.akka:akka-actor_*</exclude>
> <exclude>com.typesafe.akka:akka-remote_*</exclude>
> <exclude>com.typesafe.akka:akka-slf4j_*</exclude>
> <exclude>io.netty:netty-all</exclude>
> <exclude>io.netty:netty</exclude>
> <exclude>org.eclipse.jetty:jetty-server</exclude>
> <exclude>org.eclipse.jetty:jetty-continuation</exclude>
> <exclude>org.eclipse.jetty:jetty-http</exclude>
> <exclude>org.eclipse.jetty:jetty-io</exclude>
> <exclude>org.eclipse.jetty:jetty-util</exclude>
> <exclude>org.eclipse.jetty:jetty-security</exclude>
> <exclude>org.eclipse.jetty:jetty-servlet</exclude>
> <exclude>commons-fileupload:commons-fileupload</exclude>
> <exclude>org.apache.avro:avro</exclude>
> <exclude>commons-collections:commons-collections</exclude>
> <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
> <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
> <exclude>com.thoughtworks.paranamer:paranamer</exclude>
> <exclude>org.xerial.snappy:snappy-java</exclude>
> <exclude>org.apache.commons:commons-compress</exclude>
> <exclude>org.tukaani:xz</exclude>
> <exclude>com.esotericsoftware.kryo:kryo</exclude>
> <exclude>com.esotericsoftware.minlog:minlog</exclude>
> <exclude>org.objenesis:objenesis</exclude>
> <exclude>com.twitter:chill_*</exclude>
> <exclude>com.twitter:chill-java</exclude>
> <exclude>com.twitter:chill-avro_*</exclude>
> <exclude>com.twitter:chill-bijection_*</exclude>
> <exclude>com.twitter:bijection-core_*</exclude>
> <exclude>com.twitter:bijection-avro_*</exclude>
> <exclude>commons-lang:commons-lang</exclude>
> <exclude>junit:junit</exclude>
> <exclude>de.javakaffee:kryo-serializers</exclude>
> <exclude>joda-time:joda-time</exclude>
> <exclude>org.apache.commons:commons-lang3</exclude>
> <exclude>org.slf4j:slf4j-api</exclude>
> <exclude>org.slf4j:slf4j-log4j12</exclude>
> <exclude>log4j:log4j</exclude>
> <exclude>org.apache.commons:commons-math</exclude>
> <exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
> <exclude>commons-logging:commons-logging</exclude>
> <exclude>org.apache.httpcomponents:httpclient</exclude>
> <exclude>org.apache.httpcomponents:httpcore</exclude>
> <exclude>commons-codec:commons-codec</exclude>
> <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
> <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
> <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
> <exclude>org.codehaus.jettison:jettison</exclude>
> <exclude>stax:stax-api</exclude>
> <exclude>com.typesafe:config</exclude>
> <exclude>org.uncommons.maths:uncommons-maths</exclude>
> <exclude>com.github.scopt:scopt_*</exclude>
> <exclude>org.mortbay.jetty:servlet-api</exclude>
> <exclude>commons-io:commons-io</exclude>
> <exclude>commons-cli:commons-cli</exclude>
> </excludes>
> </artifactSet>
> <filters>
> <filter>
> <artifact>org.apache.flink:*</artifact>
> <excludes>
> <exclude>org/apache/flink/shaded/**</exclude>
> <exclude>web-docs/**</exclude>
> </excludes>
> </filter>
> <filter>
> <!-- Do not copy the signatures in the META-INF folder.
> Otherwise, this might cause SecurityExceptions when using the JAR. -->
> <artifact>*:*</artifact>
> <excludes>
> <exclude>META-INF/*.SF</exclude>
> <exclude>META-INF/*.DSA</exclude>
> <exclude>META-INF/*.RSA</exclude>
> </excludes>
> </filter>
> </filters>
> <transformers>
> <!-- add Main-Class to manifest file -->
> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
> <mainClass>com.datahub.heka.Main</mainClass>
> </transformer>
> </transformers>
> <createDependencyReducedPom>false</createDependencyReducedPom>
> </configuration>
> </execution>
> </executions>
> </plugin>
>
> <!-- Configure the jar plugin to add the main class as a manifest entry -->
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-jar-plugin</artifactId>
> <version>2.5</version>
> <configuration>
> <archive>
> <manifestEntries>
> <Main-Class>com.datahub.heka.Main</Main-Class>
> </manifestEntries>
> </archive>
> </configuration>
> </plugin>
>
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-compiler-plugin</artifactId>
> <version>3.1</version>
> <configuration>
> <source>1.8</source> <!-- If you want to use Java 8, change this to "1.8" -->
> <target>1.8</target> <!-- If you want to use Java 8, change this to "1.8" -->
> </configuration>
> </plugin>
> </plugins>
>
>
> </build>
> <profiles>
> <profile>
> <!-- A profile that does everyting correctly:
> We set the Flink dependencies to provided -->
> <id>build-jar</id>
> <activation>
> <activeByDefault>false</activeByDefault>
> </activation>
> <dependencies>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-java</artifactId>
> <version>0.10.1</version>
> <scope>provided</scope>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-streaming-java</artifactId>
> <version>0.10.1</version>
> <scope>provided</scope>
> </dependency>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-clients</artifactId>
> <version>0.10.1</version>
> <scope>provided</scope>
> </dependency>
> </dependencies>
> </profile>
> </profiles>
> </project>
>
>
> Any help is appreciated.
>
>
> Thanks
>
>