You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by dmt312_2010 <dm...@163.com> on 2020/07/15 02:48:41 UTC
flink1.11.0 java.lang.NoSuchMethodError: org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread
Hi,
大家好，请教各位大佬一个问题，我在验证flink 1.11.0时，遇到如下问题：
报错信息：
[ERROR] Could not execute SQL statement. Reason:
java.lang.NoSuchMethodError: org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.<init>(Lorg/slf4j/Logger;Lorg/apache/flink/streaming/connectors/kafka/internal/Handover;Ljava/util/Properties;
Lorg/apache/flink/streaming/connectors/kafka/internals/ClosableBlockingQueue;Ljava/lang/String;
JZLorg/apache/flink/metrics/MetricGroup;Lorg/apache/flink/metrics/MetricGroup;)V
请问下是缺少某些需要的包吗？
环境信息：
版本：flink 1.11.0
启动方式：flink on yarn (集成到CDH中)
Flink Home：
/opt/cloudera/parcels/FLINK/lib/flink
${FLINK_HOME}/lib/ jar包信息:
-rw-r--r-- 1 root root 53820 Mar 9 2017 commons-cli-1.4.jar
-rw-r--r-- 1 root root 284220 Jan 17 2011 commons-lang-2.6.jar
-rw-r--r-- 1 root root 61829 Jul 6 2014 commons-logging-1.2.jar
-rw-r--r-- 1 root root 197130 Jun 30 12:45 flink-connector-jdbc_2.12-1.11.0.jar
-rw-r--r-- 1 root root 47655 Jun 30 12:41 flink-connector-kafka-0.10_2.12-1.11.0.jar
-rw-r--r-- 1 root root 60151 Jun 30 12:42 flink-connector-kafka-0.11_2.12-1.11.0.jar
-rw-r--r-- 1 root root 109660 Jun 30 12:46 flink-connector-kafka_2.12-1.11.0.jar
-rw-r--r-- 1 root root 122794 Jun 30 12:41 flink-connector-kafka-base_2.12-1.11.0.jar
-rw-r--r-- 1 root root 90782 Jun 30 16:40 flink-csv-1.11.0.jar
-rw-r--r-- 1 root root 99461460 Jun 30 16:46 flink-dist_2.12-1.11.0.jar
-rw-r--r-- 1 root root 94863 Jun 30 16:40 flink-json-1.11.0.jar
-rw-rw-r-- 1 root root 19127 Jul 14 19:12 flink-metrics-core-1.11.0.jar
-rw-r--r-- 1 root root 108120 Jul 14 19:35 flink-metrics-prometheus-1.11.0.jar
-rw-r--r-- 1 root root 7712156 Jun 18 10:42 flink-shaded-zookeeper-3.4.14.jar
-rw-r--r-- 1 root root 31924588 Jun 30 16:45 flink-table_2.12-1.11.0.jar
-rw-r--r-- 1 root root 34817036 Jun 30 16:45 flink-table-blink_2.12-1.11.0.jar
-rw-r--r-- 1 root root 2740491 Oct 18 2019 kafka-clients-2.3.1.jar
-rw-r--r-- 1 root root 67114 Apr 20 20:47 log4j-1.2-api-2.12.1.jar
-rw-r--r-- 1 root root 276771 Apr 20 20:47 log4j-api-2.12.1.jar
-rw-r--r-- 1 root root 1674433 Apr 20 20:47 log4j-core-2.12.1.jar
-rw-r--r-- 1 root root 23518 Apr 20 20:47 log4j-slf4j-impl-2.12.1.jar
-rw-r--r-- 1 root root 1006904 Apr 20 11:10 mysql-connector-java-5.1.49.jar
-rw-r--r-- 1 root root 26084 Jul 14 18:54 slf4j-api-1.7.5.jar
-rw-r--r-- 1 root root 8869 Jul 14 18:54 slf4j-log4j12-1.7.5.jar
-rwxr-xr-x 1 root root 10680 Jul 14 18:54 slf4j-simple-1.7.5.jar
执行语句
bin/sql-client.sh embedded
CREATE TABLE TZT_PAYMENT_ORDER (
`op_type` STRING,
`op_ts` STRING,
`current_ts` STRING,
`pos` STRING,
`ID` BIGINT,
`TRACE_ID` STRING,
`BIZ_SYSTEM_CODE` STRING,
`MERCHANT_NO` STRING,
`REQUEST_NO` STRING,
`PRODUCT_NAME` STRING,
`BANK_CODE` STRING,
`CREATE_TIME` STRING,
`row_ts` as TO_TIMESTAMP(`CREATE_TIME`),
WATERMARK FOR row_ts AS row_ts - INTERVAL '1' MINUTE
) WITH (
'format' = 'json',
'connector' = 'kafka',
'topic' = 'TZT_TZT_PAYMENT_ORDER',
'properties.bootstrap.servers' = 'xxxx',
'properties.group.id' = 'TZT_TZT_PAYMENT_ORDER_TEST_WWX',
'scan.startup.mode' = 'earliest-offset'
);
select * from TZT_PAYMENT_ORDER;
Re: flink1.11.0 java.lang.NoSuchMethodError:
org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread
Posted by Leonard Xu <xb...@gmail.com>.
Hi,
两个kafka connector是会冲突的，还有一点是 SQL client 里应该用 sql 的connector依赖吧， flink-sql-connector-kafka_2.12-1.11.0.jar
祝好，
Leonard Xu
> 在 2020年7月15日，10:56，Paul Lam <pa...@gmail.com> 写道：
>
> Hi,
>
> 看起来是 Kafka connector class 冲突了，flink-connector-kafka_2.12-1.11.0.jar 和 flink-connector-kafka-0.10_2.12-1.11.0.jar 不能同时加到 classpath 里。
>
> Best,
> Paul Lam
>
>> 2020年7月15日 10:48，dmt312_2010 <dm...@163.com> 写道：
>>
>> Hi,
>> 大家好，请教各位大佬一个问题，我在验证flink 1.11.0时，遇到如下问题：
>>
>>
>> 报错信息：
>>
>>
>> [ERROR] Could not execute SQL statement. Reason:
>> java.lang.NoSuchMethodError: org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.<init>(Lorg/slf4j/Logger;Lorg/apache/flink/streaming/connectors/kafka/internal/Handover;Ljava/util/Properties;
>> Lorg/apache/flink/streaming/connectors/kafka/internals/ClosableBlockingQueue;Ljava/lang/String;
>> JZLorg/apache/flink/metrics/MetricGroup;Lorg/apache/flink/metrics/MetricGroup;)V
>>
>>
>> 请问下是缺少某些需要的包吗？
>>
>>
>>
>>
>> 环境信息：
>>
>>
>> 版本：flink 1.11.0
>> 启动方式：flink on yarn (集成到CDH中)
>>
>>
>> Flink Home：
>> /opt/cloudera/parcels/FLINK/lib/flink
>>
>>
>> ${FLINK_HOME}/lib/ jar包信息:
>>
>>
>> -rw-r--r-- 1 root root 53820 Mar 9 2017 commons-cli-1.4.jar
>> -rw-r--r-- 1 root root 284220 Jan 17 2011 commons-lang-2.6.jar
>> -rw-r--r-- 1 root root 61829 Jul 6 2014 commons-logging-1.2.jar
>> -rw-r--r-- 1 root root 197130 Jun 30 12:45 flink-connector-jdbc_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 47655 Jun 30 12:41 flink-connector-kafka-0.10_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 60151 Jun 30 12:42 flink-connector-kafka-0.11_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 109660 Jun 30 12:46 flink-connector-kafka_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 122794 Jun 30 12:41 flink-connector-kafka-base_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 90782 Jun 30 16:40 flink-csv-1.11.0.jar
>> -rw-r--r-- 1 root root 99461460 Jun 30 16:46 flink-dist_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 94863 Jun 30 16:40 flink-json-1.11.0.jar
>> -rw-rw-r-- 1 root root 19127 Jul 14 19:12 flink-metrics-core-1.11.0.jar
>> -rw-r--r-- 1 root root 108120 Jul 14 19:35 flink-metrics-prometheus-1.11.0.jar
>> -rw-r--r-- 1 root root 7712156 Jun 18 10:42 flink-shaded-zookeeper-3.4.14.jar
>> -rw-r--r-- 1 root root 31924588 Jun 30 16:45 flink-table_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 34817036 Jun 30 16:45 flink-table-blink_2.12-1.11.0.jar
>> -rw-r--r-- 1 root root 2740491 Oct 18 2019 kafka-clients-2.3.1.jar
>> -rw-r--r-- 1 root root 67114 Apr 20 20:47 log4j-1.2-api-2.12.1.jar
>> -rw-r--r-- 1 root root 276771 Apr 20 20:47 log4j-api-2.12.1.jar
>> -rw-r--r-- 1 root root 1674433 Apr 20 20:47 log4j-core-2.12.1.jar
>> -rw-r--r-- 1 root root 23518 Apr 20 20:47 log4j-slf4j-impl-2.12.1.jar
>> -rw-r--r-- 1 root root 1006904 Apr 20 11:10 mysql-connector-java-5.1.49.jar
>> -rw-r--r-- 1 root root 26084 Jul 14 18:54 slf4j-api-1.7.5.jar
>> -rw-r--r-- 1 root root 8869 Jul 14 18:54 slf4j-log4j12-1.7.5.jar
>> -rwxr-xr-x 1 root root 10680 Jul 14 18:54 slf4j-simple-1.7.5.jar
>>
>>
>>
>>
>>
>>
>> 执行语句
>>
>>
>> bin/sql-client.sh embedded
>>
>>
>>
>>
>> CREATE TABLE TZT_PAYMENT_ORDER (
>> `op_type` STRING,
>> `op_ts` STRING,
>> `current_ts` STRING,
>> `pos` STRING,
>> `ID` BIGINT,
>> `TRACE_ID` STRING,
>> `BIZ_SYSTEM_CODE` STRING,
>> `MERCHANT_NO` STRING,
>> `REQUEST_NO` STRING,
>> `PRODUCT_NAME` STRING,
>> `BANK_CODE` STRING,
>> `CREATE_TIME` STRING,
>> `row_ts` as TO_TIMESTAMP(`CREATE_TIME`),
>> WATERMARK FOR row_ts AS row_ts - INTERVAL '1' MINUTE
>> ) WITH (
>> 'format' = 'json',
>> 'connector' = 'kafka',
>> 'topic' = 'TZT_TZT_PAYMENT_ORDER',
>> 'properties.bootstrap.servers' = 'xxxx',
>> 'properties.group.id' = 'TZT_TZT_PAYMENT_ORDER_TEST_WWX',
>> 'scan.startup.mode' = 'earliest-offset'
>> );
>>
>>
>> select * from TZT_PAYMENT_ORDER;
>
Re: flink1.11.0 java.lang.NoSuchMethodError:
org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread
Posted by Paul Lam <pa...@gmail.com>.
Hi,
看起来是 Kafka connector class 冲突了，flink-connector-kafka_2.12-1.11.0.jar 和 flink-connector-kafka-0.10_2.12-1.11.0.jar 不能同时加到 classpath 里。
Best,
Paul Lam
> 2020年7月15日 10:48，dmt312_2010 <dm...@163.com> 写道：
>
> Hi,
> 大家好，请教各位大佬一个问题，我在验证flink 1.11.0时，遇到如下问题：
>
>
> 报错信息：
>
>
> [ERROR] Could not execute SQL statement. Reason:
> java.lang.NoSuchMethodError: org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.<init>(Lorg/slf4j/Logger;Lorg/apache/flink/streaming/connectors/kafka/internal/Handover;Ljava/util/Properties;
> Lorg/apache/flink/streaming/connectors/kafka/internals/ClosableBlockingQueue;Ljava/lang/String;
> JZLorg/apache/flink/metrics/MetricGroup;Lorg/apache/flink/metrics/MetricGroup;)V
>
>
> 请问下是缺少某些需要的包吗？
>
>
>
>
> 环境信息：
>
>
> 版本：flink 1.11.0
> 启动方式：flink on yarn (集成到CDH中)
>
>
> Flink Home：
> /opt/cloudera/parcels/FLINK/lib/flink
>
>
> ${FLINK_HOME}/lib/ jar包信息:
>
>
> -rw-r--r-- 1 root root 53820 Mar 9 2017 commons-cli-1.4.jar
> -rw-r--r-- 1 root root 284220 Jan 17 2011 commons-lang-2.6.jar
> -rw-r--r-- 1 root root 61829 Jul 6 2014 commons-logging-1.2.jar
> -rw-r--r-- 1 root root 197130 Jun 30 12:45 flink-connector-jdbc_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 47655 Jun 30 12:41 flink-connector-kafka-0.10_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 60151 Jun 30 12:42 flink-connector-kafka-0.11_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 109660 Jun 30 12:46 flink-connector-kafka_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 122794 Jun 30 12:41 flink-connector-kafka-base_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 90782 Jun 30 16:40 flink-csv-1.11.0.jar
> -rw-r--r-- 1 root root 99461460 Jun 30 16:46 flink-dist_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 94863 Jun 30 16:40 flink-json-1.11.0.jar
> -rw-rw-r-- 1 root root 19127 Jul 14 19:12 flink-metrics-core-1.11.0.jar
> -rw-r--r-- 1 root root 108120 Jul 14 19:35 flink-metrics-prometheus-1.11.0.jar
> -rw-r--r-- 1 root root 7712156 Jun 18 10:42 flink-shaded-zookeeper-3.4.14.jar
> -rw-r--r-- 1 root root 31924588 Jun 30 16:45 flink-table_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 34817036 Jun 30 16:45 flink-table-blink_2.12-1.11.0.jar
> -rw-r--r-- 1 root root 2740491 Oct 18 2019 kafka-clients-2.3.1.jar
> -rw-r--r-- 1 root root 67114 Apr 20 20:47 log4j-1.2-api-2.12.1.jar
> -rw-r--r-- 1 root root 276771 Apr 20 20:47 log4j-api-2.12.1.jar
> -rw-r--r-- 1 root root 1674433 Apr 20 20:47 log4j-core-2.12.1.jar
> -rw-r--r-- 1 root root 23518 Apr 20 20:47 log4j-slf4j-impl-2.12.1.jar
> -rw-r--r-- 1 root root 1006904 Apr 20 11:10 mysql-connector-java-5.1.49.jar
> -rw-r--r-- 1 root root 26084 Jul 14 18:54 slf4j-api-1.7.5.jar
> -rw-r--r-- 1 root root 8869 Jul 14 18:54 slf4j-log4j12-1.7.5.jar
> -rwxr-xr-x 1 root root 10680 Jul 14 18:54 slf4j-simple-1.7.5.jar
>
>
>
>
>
>
> 执行语句
>
>
> bin/sql-client.sh embedded
>
>
>
>
> CREATE TABLE TZT_PAYMENT_ORDER (
> `op_type` STRING,
> `op_ts` STRING,
> `current_ts` STRING,
> `pos` STRING,
> `ID` BIGINT,
> `TRACE_ID` STRING,
> `BIZ_SYSTEM_CODE` STRING,
> `MERCHANT_NO` STRING,
> `REQUEST_NO` STRING,
> `PRODUCT_NAME` STRING,
> `BANK_CODE` STRING,
> `CREATE_TIME` STRING,
> `row_ts` as TO_TIMESTAMP(`CREATE_TIME`),
> WATERMARK FOR row_ts AS row_ts - INTERVAL '1' MINUTE
> ) WITH (
> 'format' = 'json',
> 'connector' = 'kafka',
> 'topic' = 'TZT_TZT_PAYMENT_ORDER',
> 'properties.bootstrap.servers' = 'xxxx',
> 'properties.group.id' = 'TZT_TZT_PAYMENT_ORDER_TEST_WWX',
> 'scan.startup.mode' = 'earliest-offset'
> );
>
>
> select * from TZT_PAYMENT_ORDER;
Re: flink1.11.0 java.lang.NoSuchMethodError:
org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread
Posted by JasonLee <17...@163.com>.
这个很明显是jar包冲突了 只保留flink-sql那个包就行了
| |
JasonLee
|
|
邮箱：17610775726@163.com
|
Signature is customized by Netease Mail Master
On 07/15/2020 10:48, dmt312_2010 wrote:
Hi,
大家好，请教各位大佬一个问题，我在验证flink 1.11.0时，遇到如下问题：
报错信息：
[ERROR] Could not execute SQL statement. Reason:
java.lang.NoSuchMethodError: org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.<init>(Lorg/slf4j/Logger;Lorg/apache/flink/streaming/connectors/kafka/internal/Handover;Ljava/util/Properties;
Lorg/apache/flink/streaming/connectors/kafka/internals/ClosableBlockingQueue;Ljava/lang/String;
JZLorg/apache/flink/metrics/MetricGroup;Lorg/apache/flink/metrics/MetricGroup;)V
请问下是缺少某些需要的包吗？
环境信息：
版本：flink 1.11.0
启动方式：flink on yarn (集成到CDH中)
Flink Home：
/opt/cloudera/parcels/FLINK/lib/flink
${FLINK_HOME}/lib/ jar包信息:
-rw-r--r-- 1 root root 53820 Mar 9 2017 commons-cli-1.4.jar
-rw-r--r-- 1 root root 284220 Jan 17 2011 commons-lang-2.6.jar
-rw-r--r-- 1 root root 61829 Jul 6 2014 commons-logging-1.2.jar
-rw-r--r-- 1 root root 197130 Jun 30 12:45 flink-connector-jdbc_2.12-1.11.0.jar
-rw-r--r-- 1 root root 47655 Jun 30 12:41 flink-connector-kafka-0.10_2.12-1.11.0.jar
-rw-r--r-- 1 root root 60151 Jun 30 12:42 flink-connector-kafka-0.11_2.12-1.11.0.jar
-rw-r--r-- 1 root root 109660 Jun 30 12:46 flink-connector-kafka_2.12-1.11.0.jar
-rw-r--r-- 1 root root 122794 Jun 30 12:41 flink-connector-kafka-base_2.12-1.11.0.jar
-rw-r--r-- 1 root root 90782 Jun 30 16:40 flink-csv-1.11.0.jar
-rw-r--r-- 1 root root 99461460 Jun 30 16:46 flink-dist_2.12-1.11.0.jar
-rw-r--r-- 1 root root 94863 Jun 30 16:40 flink-json-1.11.0.jar
-rw-rw-r-- 1 root root 19127 Jul 14 19:12 flink-metrics-core-1.11.0.jar
-rw-r--r-- 1 root root 108120 Jul 14 19:35 flink-metrics-prometheus-1.11.0.jar
-rw-r--r-- 1 root root 7712156 Jun 18 10:42 flink-shaded-zookeeper-3.4.14.jar
-rw-r--r-- 1 root root 31924588 Jun 30 16:45 flink-table_2.12-1.11.0.jar
-rw-r--r-- 1 root root 34817036 Jun 30 16:45 flink-table-blink_2.12-1.11.0.jar
-rw-r--r-- 1 root root 2740491 Oct 18 2019 kafka-clients-2.3.1.jar
-rw-r--r-- 1 root root 67114 Apr 20 20:47 log4j-1.2-api-2.12.1.jar
-rw-r--r-- 1 root root 276771 Apr 20 20:47 log4j-api-2.12.1.jar
-rw-r--r-- 1 root root 1674433 Apr 20 20:47 log4j-core-2.12.1.jar
-rw-r--r-- 1 root root 23518 Apr 20 20:47 log4j-slf4j-impl-2.12.1.jar
-rw-r--r-- 1 root root 1006904 Apr 20 11:10 mysql-connector-java-5.1.49.jar
-rw-r--r-- 1 root root 26084 Jul 14 18:54 slf4j-api-1.7.5.jar
-rw-r--r-- 1 root root 8869 Jul 14 18:54 slf4j-log4j12-1.7.5.jar
-rwxr-xr-x 1 root root 10680 Jul 14 18:54 slf4j-simple-1.7.5.jar
执行语句
bin/sql-client.sh embedded
CREATE TABLE TZT_PAYMENT_ORDER (
`op_type` STRING,
`op_ts` STRING,
`current_ts` STRING,
`pos` STRING,
`ID` BIGINT,
`TRACE_ID` STRING,
`BIZ_SYSTEM_CODE` STRING,
`MERCHANT_NO` STRING,
`REQUEST_NO` STRING,
`PRODUCT_NAME` STRING,
`BANK_CODE` STRING,
`CREATE_TIME` STRING,
`row_ts` as TO_TIMESTAMP(`CREATE_TIME`),
WATERMARK FOR row_ts AS row_ts - INTERVAL '1' MINUTE
) WITH (
'format' = 'json',
'connector' = 'kafka',
'topic' = 'TZT_TZT_PAYMENT_ORDER',
'properties.bootstrap.servers' = 'xxxx',
'properties.group.id' = 'TZT_TZT_PAYMENT_ORDER_TEST_WWX',
'scan.startup.mode' = 'earliest-offset'
);
select * from TZT_PAYMENT_ORDER;