You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by 陈帅 <ca...@gmail.com> on 2020/09/13 03:10:10 UTC

通过sql client操作flink mysql-cdc异常

flink版本是1.11.1，我将
flink-connector-debezium-1.1.0.jar，
flink-connector-mysql-cdc-1.1.0.jar，
flink-sql-connector-kafka_2.12-1.11.1.jar，
flink-sql-connector-mysql-cdc-1.1.0.jar,
flink-format-changelog-json-1.1.0.jar
下载并加入到 $FLINK_HOME/lib目录，并以embedded模式启动flink sql
client，同时在mysql中插入一张表，然后在flink sql client中创建相应的表，语句如下

Flink SQL>  CREATE TABLE order_info(
    id BIGINT,
    user_id BIGINT,
    create_time TIMESTAMP(0),
    operate_time TIMESTAMP(0),
    province_id INT,
    order_status STRING,
    total_amount DECIMAL(10, 5)
  ) WITH (
    'connector' = 'mysql-cdc',
    'hostname' = 'localhost',
    'port' = '3306',
    'username' = 'root',
    'password' = 'xxxxx,
    'database-name' = 'test',
    'table-name' = 'order_info'
);

最后在flink sql client中执行查询
Flink SQL>  select * from order_info;

*[ERROR] Could not execute SQL statement. Reason:*

*java.lang.ClassNotFoundException:
com.alibaba.ververica.cdc.debezium.DebeziumSourceFunction*


结果报了如上exception，我查了下这个类是属于flink-connector-debezium-1.1.0.jar的，而这个jar我已经放到$FLINK_HOME/lib目录下，并且能够解压看到所报缺失类全路径，这是为什么？要如何修复？

sql client日志如下：

ClassLoader info: URL ClassLoader:
    file:
'/var/folders/7n/pfzv54s94w9d9jl578txzx200000gn/T/blobStore-ae8ce496-872b-4934-a212-405e34ecfd6f/job_a2634d29a69aa47bfdb0e65b522ff2e8/blob_p-949ecf57aaab045c3136da62fe2e2ab39f502c30-723503d095b849a5d7f3375ef6ddc85f'
(valid JAR)
Class not resolvable through given classloader.
at
org.apache.flink.streaming.api.graph.StreamConfig.getStreamOperatorFactory(StreamConfig.java:288)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.OperatorChain.<init>(OperatorChain.java:126)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:453)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:522)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:721)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:546)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_231]
Caused by: java.lang.ClassNotFoundException:
com.alibaba.ververica.cdc.debezium.DebeziumSourceFunction
at java.net.URLClassLoader.findClass(URLClassLoader.java:382) ~[?:1.8.0_231]
at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_231]
at
org.apache.flink.util.FlinkUserCodeClassLoader.loadClassWithoutExceptionHandling(FlinkUserCodeClassLoader.java:61)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.util.ChildFirstClassLoader.loadClassWithoutExceptionHandling(ChildFirstClassLoader.java:74)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.util.FlinkUserCodeClassLoader.loadClass(FlinkUserCodeClassLoader.java:48)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_231]
at java.lang.Class.forName0(Native Method) ~[?:1.8.0_231]
at java.lang.Class.forName(Class.java:348) ~[?:1.8.0_231]
at
org.apache.flink.util.InstantiationUtil$ClassLoaderObjectInputStream.resolveClass(InstantiationUtil.java:78)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1868)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751)
~[?:1.8.0_231]
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
~[?:1.8.0_231]
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2287)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
~[?:1.8.0_231]
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
~[?:1.8.0_231]
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2287)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
~[?:1.8.0_231]
at
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
~[?:1.8.0_231]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
~[?:1.8.0_231]
at
org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:576)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:562)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:550)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.util.InstantiationUtil.readObjectFromConfig(InstantiationUtil.java:511)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.api.graph.StreamConfig.getStreamOperatorFactory(StreamConfig.java:276)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.OperatorChain.<init>(OperatorChain.java:126)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:453)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:522)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:721)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:546)
~[flink-dist_2.11-1.11.1.jar:1.11.1]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_231]
2020-09-13 10:54:16,700 INFO
 org.apache.flink.runtime.checkpoint.CheckpointCoordinator    [] - Stopping
checkpoint coordinator for job a2634d29a69aa47bfdb0e65b522ff2e8.

Re: 通过sql client操作flink mysql-cdc异常

Posted by Jark Wu <im...@gmail.com>.
1. 请检查所有的 jar 包是否有破损
2. 只需保留 flink-sql-connector-mysql-cdc-1.1.0.jar,
flink-format-changelog-json-1.1.0.jar 这两个 jar 包，
    flink-connector-debezium-1.1.0.jar 和
flink-connector-mysql-cdc-1.1.0.jar 不需要。

Best,
Jark

On Sun, 13 Sep 2020 at 11:10, 陈帅 <ca...@gmail.com> wrote:

> flink版本是1.11.1，我将
> flink-connector-debezium-1.1.0.jar，
> flink-connector-mysql-cdc-1.1.0.jar，
> flink-sql-connector-kafka_2.12-1.11.1.jar，
> flink-sql-connector-mysql-cdc-1.1.0.jar,
> flink-format-changelog-json-1.1.0.jar
> 下载并加入到 $FLINK_HOME/lib目录，并以embedded模式启动flink sql
> client，同时在mysql中插入一张表，然后在flink sql client中创建相应的表，语句如下
>
> Flink SQL>  CREATE TABLE order_info(
>     id BIGINT,
>     user_id BIGINT,
>     create_time TIMESTAMP(0),
>     operate_time TIMESTAMP(0),
>     province_id INT,
>     order_status STRING,
>     total_amount DECIMAL(10, 5)
>   ) WITH (
>     'connector' = 'mysql-cdc',
>     'hostname' = 'localhost',
>     'port' = '3306',
>     'username' = 'root',
>     'password' = 'xxxxx,
>     'database-name' = 'test',
>     'table-name' = 'order_info'
> );
>
> 最后在flink sql client中执行查询
> Flink SQL>  select * from order_info;
>
> *[ERROR] Could not execute SQL statement. Reason:*
>
> *java.lang.ClassNotFoundException:
> com.alibaba.ververica.cdc.debezium.DebeziumSourceFunction*
>
>
>
> 结果报了如上exception，我查了下这个类是属于flink-connector-debezium-1.1.0.jar的，而这个jar我已经放到$FLINK_HOME/lib目录下，并且能够解压看到所报缺失类全路径，这是为什么？要如何修复？
>
> sql client日志如下：
>
> ClassLoader info: URL ClassLoader:
>     file:
>
> '/var/folders/7n/pfzv54s94w9d9jl578txzx200000gn/T/blobStore-ae8ce496-872b-4934-a212-405e34ecfd6f/job_a2634d29a69aa47bfdb0e65b522ff2e8/blob_p-949ecf57aaab045c3136da62fe2e2ab39f502c30-723503d095b849a5d7f3375ef6ddc85f'
> (valid JAR)
> Class not resolvable through given classloader.
> at
>
> org.apache.flink.streaming.api.graph.StreamConfig.getStreamOperatorFactory(StreamConfig.java:288)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.OperatorChain.<init>(OperatorChain.java:126)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:453)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:522)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:721)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:546)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_231]
> Caused by: java.lang.ClassNotFoundException:
> com.alibaba.ververica.cdc.debezium.DebeziumSourceFunction
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> ~[?:1.8.0_231]
> at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_231]
> at
>
> org.apache.flink.util.FlinkUserCodeClassLoader.loadClassWithoutExceptionHandling(FlinkUserCodeClassLoader.java:61)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.util.ChildFirstClassLoader.loadClassWithoutExceptionHandling(ChildFirstClassLoader.java:74)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.util.FlinkUserCodeClassLoader.loadClass(FlinkUserCodeClassLoader.java:48)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_231]
> at java.lang.Class.forName0(Native Method) ~[?:1.8.0_231]
> at java.lang.Class.forName(Class.java:348) ~[?:1.8.0_231]
> at
>
> org.apache.flink.util.InstantiationUtil$ClassLoaderObjectInputStream.resolveClass(InstantiationUtil.java:78)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1868)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751)
> ~[?:1.8.0_231]
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2287)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
> ~[?:1.8.0_231]
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2287)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
> ~[?:1.8.0_231]
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
> ~[?:1.8.0_231]
> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
> ~[?:1.8.0_231]
> at
>
> org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:576)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:562)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:550)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.util.InstantiationUtil.readObjectFromConfig(InstantiationUtil.java:511)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.api.graph.StreamConfig.getStreamOperatorFactory(StreamConfig.java:276)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.OperatorChain.<init>(OperatorChain.java:126)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:453)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at
>
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:522)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:721)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:546)
> ~[flink-dist_2.11-1.11.1.jar:1.11.1]
> at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_231]
> 2020-09-13 10:54:16,700 INFO
>  org.apache.flink.runtime.checkpoint.CheckpointCoordinator    [] - Stopping
> checkpoint coordinator for job a2634d29a69aa47bfdb0e65b522ff2e8.
>