You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by Z-Z <zz...@qq.com> on 2020/07/17 15:10:12 UTC
回复： Flink Cli 部署问题
Flink 1.10.0 ,taskmanager报错日志如下：
2020-07-17 15:06:43,913 ERROR org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder - Caught unexpected exception.
java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:197)
at java.io.DataInputStream.readFully(DataInputStream.java:169)
at org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
at org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
at java.lang.Thread.run(Thread.java:748)
2020-07-17 15:06:43,914 WARN org.apache.flink.streaming.api.operators.BackendRestorerProcedure - Exception while restoring keyed state backend for KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from alternative (1/1), will retry while more alternatives are available.
org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected exception.
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
at org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:197)
at java.io.DataInputStream.readFully(DataInputStream.java:169)
at org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
... 15 more
2020-07-17 15:06:43,915 INFO org.apache.kafka.clients.producer.KafkaProducer - Closing the Kafka producer with timeoutMillis = 9223372036854775807 ms.
2020-07-17 15:06:43,918 INFO org.apache.flink.runtime.taskmanager.Task - Co-Keyed-Process -> Flat Map -> Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) switched from RUNNING to FAILED.
java.lang.Exception: Exception while creating StreamOperatorStateContext.
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
at org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.flink.util.FlinkException: Could not restore keyed state backend for KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from any of the 1 provided restore options.
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
... 9 more
Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected exception.
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
... 11 more
Caused by: java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:197)
at java.io.DataInputStream.readFully(DataInputStream.java:169)
at org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
at org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
... 15 more
2020-07-17 15:06:43,919 INFO org.apache.flink.runtime.taskmanager.Task - Freeing task resources for Co-Keyed-Process -> Flat Map -> Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab).
2020-07-17 15:06:43,919 INFO org.apache.flink.runtime.taskmanager.Task - Ensuring all FileSystem streams are closed for task Co-Keyed-Process -> Flat Map -> Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) [FAILED]
2020-07-17 15:06:43,931 INFO org.apache.flink.runtime.taskexecutor.TaskExecutor - Un-registering task and sending final execution state FAILED to JobManager for task Co-Keyed-Process -> Flat Map -> Sink: Unnamed (1/1) bb8f0a84e07ef90b1e11ca2825e0efab.
2020-07-17 15:06:43,947 INFO org.apache.flink.runtime.taskmanager.Task - Attempting to cancel task Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce).
2020-07-17 15:06:43,947 INFO org.apache.flink.runtime.taskmanager.Task - Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from RUNNING to CANCELING.
2020-07-17 15:06:43,947 INFO org.apache.flink.runtime.taskmanager.Task - Triggering cancellation of task code Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce).
2020-07-17 15:06:43,949 INFO org.apache.flink.runtime.taskmanager.Task - Attempting to cancel task Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
2020-07-17 15:06:43,949 INFO org.apache.flink.runtime.taskmanager.Task - Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600) switched from RUNNING to CANCELING.
2020-07-17 15:06:43,949 INFO org.apache.flink.runtime.taskmanager.Task - Triggering cancellation of task code Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
2020-07-17 15:06:43,954 INFO org.apache.flink.runtime.taskmanager.Task - Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from CANCELING to CANCELED.
2020-07-17 15:06:43,954 INFO org.apache.flink.runtime.taskmanager.Task - Freeing task resources for Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce).
2020-07-17 15:06:43,954 INFO org.apache.flink.runtime.taskmanager.Task - Ensuring all FileSystem streams are closed for task Source: Custom Source -> Flat Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) [CANCELED]
2020-07-17 15:06:43,954 INFO org.apache.flink.runtime.taskmanager.Task - Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600) switched from CANCELING to CANCELED.
2020-07-17 15:06:43,955 INFO org.apache.flink.runtime.taskmanager.Task - Freeing task resources for Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
2020-07-17 15:06:43,954 INFO org.apache.flink.runtime.taskexecutor.TaskExecutor - Un-registering task and sending final execution state CANCELED to JobManager for task Source: Custom Source -> Flat Map (1/1) 9cb8dcd4982223adcb6f007f1ffccdce.
2020-07-17 15:06:43,962 INFO org.apache.flink.runtime.taskmanager.Task - Ensuring all FileSystem streams are closed for task Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600) [CANCELED]
2020-07-17 15:06:43,962 INFO org.apache.flink.runtime.taskexecutor.TaskExecutor - Un-registering task and sending final execution state CANCELED to JobManager for task Source: Custom Source (1/1) 00621ff5d788d00c73ccaaea04717600.
2020-07-17 15:06:44,077 WARN org.apache.kafka.clients.consumer.ConsumerConfig - The configuration 'transaction.timeout.ms' was supplied but isn't a known config.
2020-07-17 15:06:44,077 WARN org.apache.kafka.clients.consumer.ConsumerConfig - The configuration 'key.serializer' was supplied but isn't a known config.
2020-07-17 15:06:44,077 WARN org.apache.kafka.clients.consumer.ConsumerConfig - The configuration 'value.serializer' was supplied but isn't a known config.
2020-07-17 15:06:44,077 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version : 0.11.0.2
2020-07-17 15:06:44,077 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId : 73be1e1168f91ee2
2020-07-17 15:06:44,077 WARN org.apache.kafka.common.utils.AppInfoParser - Error registering AppInfo mbean
javax.management.InstanceAlreadyExistsException: kafka.consumer:type=app-info,id=consumer-3
at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
at org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:757)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:633)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:615)
at org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
at org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
2020-07-17 15:06:44,079 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version : 0.11.0.2
2020-07-17 15:06:44,079 INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId : 73be1e1168f91ee2
2020-07-17 15:06:44,079 WARN org.apache.kafka.common.utils.AppInfoParser - Error registering AppInfo mbean
javax.management.InstanceAlreadyExistsException: kafka.consumer:type=app-info,id=consumer-4
at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
at org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:757)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:633)
at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:615)
at org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
at org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
------------------ 原始邮件 ------------------
发件人: "user-zh" <qcx978132955@gmail.com>;
发送时间: 2020年7月17日(星期五) 晚上10:52
收件人: "user-zh"<user-zh@flink.apache.org>;
主题: Re: Flink Cli 部署问题
Hi
请问你使用哪个版本的 Flink 呢？能否分享一下 Co-Process (1/1)
(d0309f26a545e74643382ed3f758269b) 这个 tm 的 log 呢？从上面给的日志看，应该是在 083f69d029de
这台机器上。
Best,
Congxian
Z-Z <zz9876543210@qq.com> 于2020年7月17日周五 下午6:22写道：
> 大家好，我在部署的时候发现了一个问题，我通过restAPI接口停掉了一个任务并保存了它的savepoint(步骤：/jobs/overview
> ---&gt; /jobs/{jobid}/savepoints ---&gt;
> /jobs/{jobid}/savepoints/{triggerid})，但我通过flink命令带上savepoint部署任务时会报错，但通过webui上传jar并带上savepoint就不会报错，报错堆栈如下：
> 2020-07-17 09:51:48,925 INFO&nbsp;
> org.apache.flink.runtime.resourcemanager.StandaloneResourceManager&nbsp; -
> Request slot with profile ResourceProfile{UNKNOWN} for job
> 7639673873b707aa86c4387aa7b4aac3 with allocation id
> e8865cdbfe4c3c33099c7112bc2e3231.
> 2020-07-17 09:51:48,952 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source -&gt; Filter (1/1)
> (1177659bff014e8dbc3f0508055d4307) switched from SCHEDULED to DEPLOYING.
> 2020-07-17 09:51:48,952 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Deploying Source: Custom Source -&gt; Filter (1/1) (attempt #0) to
> e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> 2020-07-17 09:51:48,953 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source (1/1) (141f0dc22b624b39e21127f637ba63c2)
> switched from SCHEDULED to DEPLOYING.
> 2020-07-17 09:51:48,953 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> 2020-07-17 09:51:48,954 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source (1/1) (274b3df03e1fab627059c1a78e4a26da)
> switched from SCHEDULED to DEPLOYING.
> 2020-07-17 09:51:48,954 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> 2020-07-17 09:51:48,954 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b) switched from
> SCHEDULED to DEPLOYING.
> 2020-07-17 09:51:48,954 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Deploying Co-Process (1/1) (attempt #0) to
> e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> 2020-07-17 09:51:48,955 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Co-Process -&gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> (618b75fcf5ea05fb5c6487bec6426e31) switched from SCHEDULED to DEPLOYING.
> 2020-07-17 09:51:48,955 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Deploying Co-Process -&gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> (attempt #0) to e63d829deafc144cd82efd73979dd056 @ 083f69d029de
> (dataPort=35758)
> 2020-07-17 09:51:49,346 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Co-Process -&gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> (618b75fcf5ea05fb5c6487bec6426e31) switched from DEPLOYING to RUNNING.
> 2020-07-17 09:51:49,370 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source (1/1) (274b3df03e1fab627059c1a78e4a26da)
> switched from DEPLOYING to RUNNING.
> 2020-07-17 09:51:49,370 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source (1/1) (141f0dc22b624b39e21127f637ba63c2)
> switched from DEPLOYING to RUNNING.
> 2020-07-17 09:51:49,377 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b) switched from
> DEPLOYING to RUNNING.
> 2020-07-17 09:51:49,377 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Source: Custom Source -&gt; Filter (1/1)
> (1177659bff014e8dbc3f0508055d4307) switched from DEPLOYING to RUNNING.
> 2020-07-17 09:51:49,493 INFO&nbsp;
> org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp; &nbsp; &nbsp;
> &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b) switched from
> RUNNING to FAILED.
> java.lang.Exception: Exception while creating StreamOperatorStateContext.
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.flink.util.FlinkException: Could not restore keyed
> state backend for
> LegacyKeyedCoProcessOperator_65e7116c7aa972ad18a796ae22bd6327_(1/1) from
> any of the 1 provided restore options.
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> ... 9 more
> Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught
> unexpected exception.
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> ... 11 more
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> ... 15 more
回复： Flink Cli 部署问题
Posted by Z-Z <zz...@qq.com>.
谢谢回复：
之前的savepoint都是通过RocksDBStateBackend生成的；
这个savepoint我通过webui 提交任务就没问题，你是说在IDE上调试savepoint吗
------------------ 原始邮件 ------------------
发件人: "user-zh" <qcx978132955@gmail.com>;
发送时间: 2020年7月19日(星期天) 晚上8:22
收件人: "user-zh"<user-zh@flink.apache.org>;
主题: Re: Flink Cli 部署问题
Hi
从你给的这部分日志看，是恢复的时候遇到 EOF 了，这个比较奇怪
1 你之前的 savepoint 是使用 RocksDBStateBackend 生成的吗
2 你还有之前在 DFS 上的 savepoint 文件吗？可能需要结合 DFS 上的文件一起看一下这个问题怎么来的
Best,
Congxian
Z-Z <zz9876543210@qq.com> 于2020年7月17日周五 下午11:10写道：
> Flink 1.10.0 ,taskmanager报错日志如下：
>
>
> 2020-07-17 15:06:43,913 ERROR
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder&nbsp;
> - Caught unexpected exception.
> java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> 2020-07-17 15:06:43,914 WARN&nbsp;
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure&nbsp; -
> Exception while restoring keyed state backend for
> KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from
> alternative (1/1), will retry while more alternatives are available.
> org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected
> exception.
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> ... 15 more
> 2020-07-17 15:06:43,915 INFO&nbsp;
> org.apache.kafka.clients.producer.KafkaProducer&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp;- Closing the Kafka producer with timeoutMillis
> = 9223372036854775807 ms.
> 2020-07-17 15:06:43,918 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Co-Keyed-Process -&gt; Flat Map
> -&gt; Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) switched from
> RUNNING to FAILED.
> java.lang.Exception: Exception while creating StreamOperatorStateContext.
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.flink.util.FlinkException: Could not restore keyed
> state backend for
> KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from any of
> the 1 provided restore options.
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> ... 9 more
> Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught
> unexpected exception.
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> ... 11 more
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> ... 15 more
> 2020-07-17 15:06:43,919 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Freeing task resources for
> Co-Keyed-Process -&gt; Flat Map -&gt; Sink: Unnamed (1/1)
> (bb8f0a84e07ef90b1e11ca2825e0efab).
> 2020-07-17 15:06:43,919 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Ensuring all FileSystem streams
> are closed for task Co-Keyed-Process -&gt; Flat Map -&gt; Sink: Unnamed
> (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) [FAILED]
> 2020-07-17 15:06:43,931 INFO&nbsp;
> org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - Un-registering task and sending final execution
> state FAILED to JobManager for task Co-Keyed-Process -&gt; Flat Map -&gt;
> Sink: Unnamed (1/1) bb8f0a84e07ef90b1e11ca2825e0efab.
> 2020-07-17 15:06:43,947 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Attempting to cancel task
> Source: Custom Source -&gt; Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,947 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Source: Custom Source -&gt; Flat
> Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from RUNNING to
> CANCELING.
> 2020-07-17 15:06:43,947 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Triggering cancellation of task
> code Source: Custom Source -&gt; Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,949 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Attempting to cancel task
> Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,949 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) switched from RUNNING to CANCELING.
> 2020-07-17 15:06:43,949 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Triggering cancellation of task
> code Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,954 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Source: Custom Source -&gt; Flat
> Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from CANCELING to
> CANCELED.
> 2020-07-17 15:06:43,954 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Freeing task resources for
> Source: Custom Source -&gt; Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,954 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Ensuring all FileSystem streams
> are closed for task Source: Custom Source -&gt; Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce) [CANCELED]
> 2020-07-17 15:06:43,954 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) switched from CANCELING to CANCELED.
> 2020-07-17 15:06:43,955 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Freeing task resources for
> Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,954 INFO&nbsp;
> org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - Un-registering task and sending final execution
> state CANCELED to JobManager for task Source: Custom Source -&gt; Flat Map
> (1/1) 9cb8dcd4982223adcb6f007f1ffccdce.
> 2020-07-17 15:06:43,962 INFO&nbsp;
> org.apache.flink.runtime.taskmanager.Task&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Ensuring all FileSystem streams
> are closed for task Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) [CANCELED]
> 2020-07-17 15:06:43,962 INFO&nbsp;
> org.apache.flink.runtime.taskexecutor.TaskExecutor&nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - Un-registering task and sending final execution
> state CANCELED to JobManager for task Source: Custom Source (1/1)
> 00621ff5d788d00c73ccaaea04717600.
> 2020-07-17 15:06:44,077 WARN&nbsp;
> org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - The configuration 'transaction.timeout.ms' was
> supplied but isn't a known config.
> 2020-07-17 15:06:44,077 WARN&nbsp;
> org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - The configuration 'key.serializer' was supplied but
> isn't a known config.
> 2020-07-17 15:06:44,077 WARN&nbsp;
> org.apache.kafka.clients.consumer.ConsumerConfig&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; - The configuration 'value.serializer' was supplied
> but isn't a known config.
> 2020-07-17 15:06:44,077 INFO&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Kafka version : 0.11.0.2
> 2020-07-17 15:06:44,077 INFO&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Kafka commitId : 73be1e1168f91ee2
> 2020-07-17 15:06:44,077 WARN&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Error registering AppInfo mbean
> javax.management.InstanceAlreadyExistsException:
> kafka.consumer:type=app-info,id=consumer-3
> at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
> at
> org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:757)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:633)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:615)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
> 2020-07-17 15:06:44,079 INFO&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Kafka version : 0.11.0.2
> 2020-07-17 15:06:44,079 INFO&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Kafka commitId : 73be1e1168f91ee2
> 2020-07-17 15:06:44,079 WARN&nbsp;
> org.apache.kafka.common.utils.AppInfoParser&nbsp; &nbsp; &nbsp; &nbsp;
> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;- Error registering AppInfo mbean
> javax.management.InstanceAlreadyExistsException:
> kafka.consumer:type=app-info,id=consumer-4
> at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
> at
> org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:757)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:633)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init&gt;(KafkaConsumer.java:615)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
>
>
>
>
>
> ------------------&nbsp;原始邮件&nbsp;------------------
> 发件人:
> "user-zh"
> <
> qcx978132955@gmail.com&gt;;
> 发送时间:&nbsp;2020年7月17日(星期五) 晚上10:52
> 收件人:&nbsp;"user-zh"<user-zh@flink.apache.org&gt;;
>
> 主题:&nbsp;Re: Flink Cli 部署问题
>
>
>
> Hi
>
> 请问你使用哪个版本的 Flink 呢？能否分享一下&nbsp; Co-Process (1/1)
> (d0309f26a545e74643382ed3f758269b) 这个 tm 的 log 呢？从上面给的日志看，应该是在 083f69d029de
> 这台机器上。
>
> Best,
> Congxian
>
>
> Z-Z <zz9876543210@qq.com&gt; 于2020年7月17日周五 下午6:22写道：
>
> &gt;
> 大家好，我在部署的时候发现了一个问题，我通过restAPI接口停掉了一个任务并保存了它的savepoint(步骤：/jobs/overview
> &gt; ---&amp;gt; /jobs/{jobid}/savepoints ---&amp;gt;
> &gt;
> /jobs/{jobid}/savepoints/{triggerid})，但我通过flink命令带上savepoint部署任务时会报错，但通过webui上传jar并带上savepoint就不会报错，报错堆栈如下：
> &gt; 2020-07-17 09:51:48,925 INFO&amp;nbsp;
> &gt;
> org.apache.flink.runtime.resourcemanager.StandaloneResourceManager&amp;nbsp;
> -
> &gt; Request slot with profile ResourceProfile{UNKNOWN} for job
> &gt; 7639673873b707aa86c4387aa7b4aac3 with allocation id
> &gt; e8865cdbfe4c3c33099c7112bc2e3231.
> &gt; 2020-07-17 09:51:48,952 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source -&amp;gt; Filter (1/1)
> &gt; (1177659bff014e8dbc3f0508055d4307) switched from SCHEDULED to
> DEPLOYING.
> &gt; 2020-07-17 09:51:48,952 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Deploying Source: Custom Source -&amp;gt; Filter (1/1)
> (attempt #0) to
> &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> &gt; 2020-07-17 09:51:48,953 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source (1/1)
> (141f0dc22b624b39e21127f637ba63c2)
> &gt; switched from SCHEDULED to DEPLOYING.
> &gt; 2020-07-17 09:51:48,953 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source (1/1)
> (274b3df03e1fab627059c1a78e4a26da)
> &gt; switched from SCHEDULED to DEPLOYING.
> &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> &gt; SCHEDULED to DEPLOYING.
> &gt; 2020-07-17 09:51:48,954 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Deploying Co-Process (1/1) (attempt #0) to
> &gt; e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> &gt; 2020-07-17 09:51:48,955 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Co-Process -&amp;gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> &gt; (618b75fcf5ea05fb5c6487bec6426e31) switched from SCHEDULED to
> DEPLOYING.
> &gt; 2020-07-17 09:51:48,955 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Deploying Co-Process -&amp;gt; (Sink: Unnamed, Sink:
> Unnamed) (1/1)
> &gt; (attempt #0) to e63d829deafc144cd82efd73979dd056 @ 083f69d029de
> &gt; (dataPort=35758)
> &gt; 2020-07-17 09:51:49,346 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Co-Process -&amp;gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> &gt; (618b75fcf5ea05fb5c6487bec6426e31) switched from DEPLOYING to RUNNING.
> &gt; 2020-07-17 09:51:49,370 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source (1/1)
> (274b3df03e1fab627059c1a78e4a26da)
> &gt; switched from DEPLOYING to RUNNING.
> &gt; 2020-07-17 09:51:49,370 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source (1/1)
> (141f0dc22b624b39e21127f637ba63c2)
> &gt; switched from DEPLOYING to RUNNING.
> &gt; 2020-07-17 09:51:49,377 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> &gt; DEPLOYING to RUNNING.
> &gt; 2020-07-17 09:51:49,377 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Source: Custom Source -&amp;gt; Filter (1/1)
> &gt; (1177659bff014e8dbc3f0508055d4307) switched from DEPLOYING to RUNNING.
> &gt; 2020-07-17 09:51:49,493 INFO&amp;nbsp;
> &gt; org.apache.flink.runtime.executiongraph.ExecutionGraph&amp;nbsp;
> &amp;nbsp; &amp;nbsp;
> &gt; &amp;nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> &gt; RUNNING to FAILED.
> &gt; java.lang.Exception: Exception while creating
> StreamOperatorStateContext.
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> java.lang.Thread.run(Thread.java:748)
> &gt; Caused by: org.apache.flink.util.FlinkException: Could not restore
> keyed
> &gt; state backend for
> &gt; LegacyKeyedCoProcessOperator_65e7116c7aa972ad18a796ae22bd6327_(1/1)
> from
> &gt; any of the 1 provided restore options.
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ... 9 more
> &gt; Caused by: org.apache.flink.runtime.state.BackendBuildingException:
> Caught
> &gt; unexpected exception.
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ... 11 more
> &gt; Caused by: java.io.EOFException
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> java.io.DataInputStream.readFully(DataInputStream.java:197)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> java.io.DataInputStream.readFully(DataInputStream.java:169)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; at
> &gt;
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> &gt;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ... 15 more
Re: Flink Cli 部署问题
Posted by Congxian Qiu <qc...@gmail.com>.
Hi
从你给的这部分日志看，是恢复的时候遇到 EOF 了，这个比较奇怪
1 你之前的 savepoint 是使用 RocksDBStateBackend 生成的吗
2 你还有之前在 DFS 上的 savepoint 文件吗？可能需要结合 DFS 上的文件一起看一下这个问题怎么来的
Best,
Congxian
Z-Z <zz...@qq.com> 于2020年7月17日周五 下午11:10写道：
> Flink 1.10.0 ,taskmanager报错日志如下：
>
>
> 2020-07-17 15:06:43,913 ERROR
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder
> - Caught unexpected exception.
> java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> 2020-07-17 15:06:43,914 WARN
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure -
> Exception while restoring keyed state backend for
> KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from
> alternative (1/1), will retry while more alternatives are available.
> org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected
> exception.
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> ... 15 more
> 2020-07-17 15:06:43,915 INFO
> org.apache.kafka.clients.producer.KafkaProducer
> - Closing the Kafka producer with timeoutMillis
> = 9223372036854775807 ms.
> 2020-07-17 15:06:43,918 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Co-Keyed-Process -> Flat Map
> -> Sink: Unnamed (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) switched from
> RUNNING to FAILED.
> java.lang.Exception: Exception while creating StreamOperatorStateContext.
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
> at
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.flink.util.FlinkException: Could not restore keyed
> state backend for
> KeyedCoProcessOperator_00360b8021b192d84949201d4fea80f2_(1/1) from any of
> the 1 provided restore options.
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> ... 9 more
> Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught
> unexpected exception.
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> at
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> at
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> at
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> ... 11 more
> Caused by: java.io.EOFException
> at java.io.DataInputStream.readFully(DataInputStream.java:197)
> at java.io.DataInputStream.readFully(DataInputStream.java:169)
> at
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> at
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> at
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> ... 15 more
> 2020-07-17 15:06:43,919 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Freeing task resources for
> Co-Keyed-Process -> Flat Map -> Sink: Unnamed (1/1)
> (bb8f0a84e07ef90b1e11ca2825e0efab).
> 2020-07-17 15:06:43,919 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Ensuring all FileSystem streams
> are closed for task Co-Keyed-Process -> Flat Map -> Sink: Unnamed
> (1/1) (bb8f0a84e07ef90b1e11ca2825e0efab) [FAILED]
> 2020-07-17 15:06:43,931 INFO
> org.apache.flink.runtime.taskexecutor.TaskExecutor
> - Un-registering task and sending final execution
> state FAILED to JobManager for task Co-Keyed-Process -> Flat Map ->
> Sink: Unnamed (1/1) bb8f0a84e07ef90b1e11ca2825e0efab.
> 2020-07-17 15:06:43,947 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Attempting to cancel task
> Source: Custom Source -> Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,947 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Source: Custom Source -> Flat
> Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from RUNNING to
> CANCELING.
> 2020-07-17 15:06:43,947 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Triggering cancellation of task
> code Source: Custom Source -> Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,949 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Attempting to cancel task
> Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,949 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) switched from RUNNING to CANCELING.
> 2020-07-17 15:06:43,949 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Triggering cancellation of task
> code Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,954 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Source: Custom Source -> Flat
> Map (1/1) (9cb8dcd4982223adcb6f007f1ffccdce) switched from CANCELING to
> CANCELED.
> 2020-07-17 15:06:43,954 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Freeing task resources for
> Source: Custom Source -> Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce).
> 2020-07-17 15:06:43,954 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Ensuring all FileSystem streams
> are closed for task Source: Custom Source -> Flat Map (1/1)
> (9cb8dcd4982223adcb6f007f1ffccdce) [CANCELED]
> 2020-07-17 15:06:43,954 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) switched from CANCELING to CANCELED.
> 2020-07-17 15:06:43,955 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Freeing task resources for
> Source: Custom Source (1/1) (00621ff5d788d00c73ccaaea04717600).
> 2020-07-17 15:06:43,954 INFO
> org.apache.flink.runtime.taskexecutor.TaskExecutor
> - Un-registering task and sending final execution
> state CANCELED to JobManager for task Source: Custom Source -> Flat Map
> (1/1) 9cb8dcd4982223adcb6f007f1ffccdce.
> 2020-07-17 15:06:43,962 INFO
> org.apache.flink.runtime.taskmanager.Task
> - Ensuring all FileSystem streams
> are closed for task Source: Custom Source (1/1)
> (00621ff5d788d00c73ccaaea04717600) [CANCELED]
> 2020-07-17 15:06:43,962 INFO
> org.apache.flink.runtime.taskexecutor.TaskExecutor
> - Un-registering task and sending final execution
> state CANCELED to JobManager for task Source: Custom Source (1/1)
> 00621ff5d788d00c73ccaaea04717600.
> 2020-07-17 15:06:44,077 WARN
> org.apache.kafka.clients.consumer.ConsumerConfig
> - The configuration 'transaction.timeout.ms' was
> supplied but isn't a known config.
> 2020-07-17 15:06:44,077 WARN
> org.apache.kafka.clients.consumer.ConsumerConfig
> - The configuration 'key.serializer' was supplied but
> isn't a known config.
> 2020-07-17 15:06:44,077 WARN
> org.apache.kafka.clients.consumer.ConsumerConfig
> - The configuration 'value.serializer' was supplied
> but isn't a known config.
> 2020-07-17 15:06:44,077 INFO
> org.apache.kafka.common.utils.AppInfoParser
> - Kafka version : 0.11.0.2
> 2020-07-17 15:06:44,077 INFO
> org.apache.kafka.common.utils.AppInfoParser
> - Kafka commitId : 73be1e1168f91ee2
> 2020-07-17 15:06:44,077 WARN
> org.apache.kafka.common.utils.AppInfoParser
> - Error registering AppInfo mbean
> javax.management.InstanceAlreadyExistsException:
> kafka.consumer:type=app-info,id=consumer-3
> at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
> at
> org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:757)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:633)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:615)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
> 2020-07-17 15:06:44,079 INFO
> org.apache.kafka.common.utils.AppInfoParser
> - Kafka version : 0.11.0.2
> 2020-07-17 15:06:44,079 INFO
> org.apache.kafka.common.utils.AppInfoParser
> - Kafka commitId : 73be1e1168f91ee2
> 2020-07-17 15:06:44,079 WARN
> org.apache.kafka.common.utils.AppInfoParser
> - Error registering AppInfo mbean
> javax.management.InstanceAlreadyExistsException:
> kafka.consumer:type=app-info,id=consumer-4
> at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
> at
> com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
> at
> org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:58)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:757)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:633)
> at
> org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:615)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.getConsumer(KafkaConsumerThread.java:502)
> at
> org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread.run(KafkaConsumerThread.java:181)
>
>
>
>
>
> ------------------ 原始邮件 ------------------
> 发件人:
> "user-zh"
> <
> qcx978132955@gmail.com>;
> 发送时间: 2020年7月17日(星期五) 晚上10:52
> 收件人: "user-zh"<user-zh@flink.apache.org>;
>
> 主题: Re: Flink Cli 部署问题
>
>
>
> Hi
>
> 请问你使用哪个版本的 Flink 呢？能否分享一下 Co-Process (1/1)
> (d0309f26a545e74643382ed3f758269b) 这个 tm 的 log 呢？从上面给的日志看，应该是在 083f69d029de
> 这台机器上。
>
> Best,
> Congxian
>
>
> Z-Z <zz9876543210@qq.com> 于2020年7月17日周五 下午6:22写道：
>
> >
> 大家好，我在部署的时候发现了一个问题，我通过restAPI接口停掉了一个任务并保存了它的savepoint(步骤：/jobs/overview
> > ---&gt; /jobs/{jobid}/savepoints ---&gt;
> >
> /jobs/{jobid}/savepoints/{triggerid})，但我通过flink命令带上savepoint部署任务时会报错，但通过webui上传jar并带上savepoint就不会报错，报错堆栈如下：
> > 2020-07-17 09:51:48,925 INFO&nbsp;
> >
> org.apache.flink.runtime.resourcemanager.StandaloneResourceManager&nbsp;
> -
> > Request slot with profile ResourceProfile{UNKNOWN} for job
> > 7639673873b707aa86c4387aa7b4aac3 with allocation id
> > e8865cdbfe4c3c33099c7112bc2e3231.
> > 2020-07-17 09:51:48,952 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source -&gt; Filter (1/1)
> > (1177659bff014e8dbc3f0508055d4307) switched from SCHEDULED to
> DEPLOYING.
> > 2020-07-17 09:51:48,952 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Deploying Source: Custom Source -&gt; Filter (1/1)
> (attempt #0) to
> > e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> > 2020-07-17 09:51:48,953 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source (1/1)
> (141f0dc22b624b39e21127f637ba63c2)
> > switched from SCHEDULED to DEPLOYING.
> > 2020-07-17 09:51:48,953 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> > e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> > 2020-07-17 09:51:48,954 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source (1/1)
> (274b3df03e1fab627059c1a78e4a26da)
> > switched from SCHEDULED to DEPLOYING.
> > 2020-07-17 09:51:48,954 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Deploying Source: Custom Source (1/1) (attempt #0) to
> > e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> > 2020-07-17 09:51:48,954 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> > SCHEDULED to DEPLOYING.
> > 2020-07-17 09:51:48,954 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Deploying Co-Process (1/1) (attempt #0) to
> > e63d829deafc144cd82efd73979dd056 @ 083f69d029de (dataPort=35758)
> > 2020-07-17 09:51:48,955 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Co-Process -&gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> > (618b75fcf5ea05fb5c6487bec6426e31) switched from SCHEDULED to
> DEPLOYING.
> > 2020-07-17 09:51:48,955 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Deploying Co-Process -&gt; (Sink: Unnamed, Sink:
> Unnamed) (1/1)
> > (attempt #0) to e63d829deafc144cd82efd73979dd056 @ 083f69d029de
> > (dataPort=35758)
> > 2020-07-17 09:51:49,346 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Co-Process -&gt; (Sink: Unnamed, Sink: Unnamed) (1/1)
> > (618b75fcf5ea05fb5c6487bec6426e31) switched from DEPLOYING to RUNNING.
> > 2020-07-17 09:51:49,370 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source (1/1)
> (274b3df03e1fab627059c1a78e4a26da)
> > switched from DEPLOYING to RUNNING.
> > 2020-07-17 09:51:49,370 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source (1/1)
> (141f0dc22b624b39e21127f637ba63c2)
> > switched from DEPLOYING to RUNNING.
> > 2020-07-17 09:51:49,377 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> > DEPLOYING to RUNNING.
> > 2020-07-17 09:51:49,377 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Source: Custom Source -&gt; Filter (1/1)
> > (1177659bff014e8dbc3f0508055d4307) switched from DEPLOYING to RUNNING.
> > 2020-07-17 09:51:49,493 INFO&nbsp;
> > org.apache.flink.runtime.executiongraph.ExecutionGraph&nbsp;
> &nbsp; &nbsp;
> > &nbsp; - Co-Process (1/1) (d0309f26a545e74643382ed3f758269b)
> switched from
> > RUNNING to FAILED.
> > java.lang.Exception: Exception while creating
> StreamOperatorStateContext.
> > at
> >
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:191)
> > at
> >
> org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:255)
> > at
> >
> org.apache.flink.streaming.runtime.tasks.StreamTask.initializeStateAndOpen(StreamTask.java:1006)
> > at
> >
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$beforeInvoke$0(StreamTask.java:454)
> > at
> >
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:94)
> > at
> >
> org.apache.flink.streaming.runtime.tasks.StreamTask.beforeInvoke(StreamTask.java:449)
> > at
> >
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:461)
> > at
> org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:707)
> > at
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:532)
> > at
> java.lang.Thread.run(Thread.java:748)
> > Caused by: org.apache.flink.util.FlinkException: Could not restore
> keyed
> > state backend for
> > LegacyKeyedCoProcessOperator_65e7116c7aa972ad18a796ae22bd6327_(1/1)
> from
> > any of the 1 provided restore options.
> > at
> >
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
> > at
> >
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:304)
> > at
> >
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:131)
> > ... 9 more
> > Caused by: org.apache.flink.runtime.state.BackendBuildingException:
> Caught
> > unexpected exception.
> > at
> >
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:336)
> > at
> >
> org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:548)
> > at
> >
> org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:288)
> > at
> >
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
> > at
> >
> org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
> > ... 11 more
> > Caused by: java.io.EOFException
> > at
> java.io.DataInputStream.readFully(DataInputStream.java:197)
> > at
> java.io.DataInputStream.readFully(DataInputStream.java:169)
> > at
> >
> org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer.deserialize(BytePrimitiveArraySerializer.java:85)
> > at
> >
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKVStateData(RocksDBFullRestoreOperation.java:221)
> > at
> >
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restoreKeyGroupsInStateHandle(RocksDBFullRestoreOperation.java:168)
> > at
> >
> org.apache.flink.contrib.streaming.state.restore.RocksDBFullRestoreOperation.restore(RocksDBFullRestoreOperation.java:151)
> > at
> >
> org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:279)
> > ... 15 more