You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by 欧阳苗 <a9...@qq.com> on 2020/04/13 02:39:20 UTC
flink java.util.concurrent.TimeoutException
job运行了两天就挂了，然后抛出如下异常，但是taskManager没有挂，其他的job还能正常在上面跑,请问这个问题是什么原因导致的，有什么好的解决办法吗
2020-04-13 06:20:31.379 ERROR 1 --- [ent-IO-thread-3] org.apache.flink.runtime.rest.RestClient.parseResponse:393 : Received response was neither of the expected type ([simple type, class org.apache.flink.runtime.rest.messages.job.JobExecutionResultResponseBody]) nor an error. Response=JsonResponse{json={"status":{"id":"COMPLETED"},"job-execution-result":{"id":"2d2a0b4efc8c3d973e2e9490b7b3b2f1","application-status":"FAILED","accumulator-results":{},"net-runtime":217272900,"failure-cause":{"class":"java.util.concurrent.TimeoutException","stack-trace":"java.util.concurrent.TimeoutException: Heartbeat of TaskManager with id 0a4ea651244982ef4b4b7092d18de776 timed out.\n\tat org.apache.flink.runtime.jobmaster.JobMaster$TaskManagerHeartbeatListener.notifyHeartbeatTimeout(JobMaster.java:1656)\n\tat org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl$HeartbeatMonitor.run(HeartbeatManagerImpl.java:339)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154)\n\tat akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)\n\tat akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:415)\n\tat scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n","serialized-throwable":"rO0ABXNyAClvcmcuYXBhY2hlLmZsaW5rLnV0aWwuU2VyaWFsaXplZFRocm93YWJsZWUWnfUfpxPzAgADTAAZZnVsbFN0cmluZ2lmaWVkU3RhY2tUcmFjZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAFm9yaWdpbmFsRXJyb3JDbGFzc05hbWVxAH4AAVsAE3NlcmlhbGl6ZWRFeGNlcHRpb250AAJbQnhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdlcQB+AAFbAApzdGFja1RyYWNldAAeW0xqYXZhL2xhbmcvU3RhY2tUcmFjZUVsZW1lbnQ7TAAUc3VwcHJlc3NlZEV4Y2VwdGlvbnN0ABBMamF2YS91dGlsL0xpc3Q7eHBwdABMSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LnVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAC3NyABtqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnRhCcWaJjbdhQIABEkACmxpbmVOdW1iZXJMAA5kZWNsYXJpbmdDbGFzc3EAfgABTAAIZmlsZU5hbWVxAH4AAUwACm1ldGhvZE5hbWVxAH4AAXhwAAAGeHQASW9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5qb2JtYXN0ZXIuSm9iTWFzdGVyJFRhc2tNYW5hZ2VySGVhcnRiZWF0TGlzdGVuZXJ0AA5Kb2JNYXN0ZXIuamF2YXQAFm5vdGlmeUhlYXJ0YmVhdFRpbWVvdXRzcQB+AAwAAAFTdABIb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmhlYXJ0YmVhdC5IZWFydGJlYXRNYW5hZ2VySW1wbCRIZWFydGJlYXRNb25pdG9ydAAZSGVhcnRiZWF0TWFuYWdlckltcGwuamF2YXQAA3J1bnNxAH4ADAAAAf90AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmF0AARjYWxsc3EAfgAMAAABCnQAH2phdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2t0AA9GdXR1cmVUYXNrLmphdmFxAH4AFHNxAH4ADAAAAJp0AGBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2t0AChBY3RvclN5c3RlbVNjaGVkdWxlZEV4ZWN1dG9yQWRhcHRlci5qYXZhcQB+ABRzcQB+AAwAAAAndAAcYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbnQAGEFic3RyYWN0RGlzcGF0Y2hlci5zY2FsYXEAfgAUc3EAfgAMAAABn3QAO2Fra2EuZGlzcGF0Y2guRm9ya0pvaW5FeGVjdXRvckNvbmZpZ3VyYXRvciRBa2thRm9ya0pvaW5UYXNrcQB+ACF0AARleGVjc3EAfgAMAAABBHQAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5UYXNrdAARRm9ya0pvaW5UYXNrLmphdmF0AAZkb0V4ZWNzcQB+AAwAAAU7dAAwc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2wkV29ya1F1ZXVldAARRm9ya0pvaW5Qb29sLmphdmF0AAdydW5UYXNrc3EAfgAMAAAHu3QAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29scQB+ACt0AAlydW5Xb3JrZXJzcQB+AAwAAABrdAAuc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pbldvcmtlclRocmVhZHQAGUZvcmtKb2luV29ya2VyVGhyZWFkLmphdmFxAH4AFHNyACZqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTGlzdPwPJTG17I4QAgABTAAEbGlzdHEAfgAHeHIALGphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVDb2xsZWN0aW9uGUIAgMte9x4CAAFMAAFjdAAWTGphdmEvdXRpbC9Db2xsZWN0aW9uO3hwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4cQB+ADh4dARkamF2YS51dGlsLmNvbmN1cnJlbnQuVGltZW91dEV4Y2VwdGlvbjogSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LgoJYXQgb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lci5ub3RpZnlIZWFydGJlYXRUaW1lb3V0KEpvYk1hc3Rlci5qYXZhOjE2NTYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3IucnVuKEhlYXJ0YmVhdE1hbmFnZXJJbXBsLmphdmE6MzM5KQoJYXQgamF2YS51dGlsLmNvbmN1cnJlbnQuRXhlY3V0b3JzJFJ1bm5hYmxlQWRhcHRlci5jYWxsKEV4ZWN1dG9ycy5qYXZhOjUxMSkKCWF0IGphdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2sucnVuKEZ1dHVyZVRhc2suamF2YToyNjYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2sucnVuKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmE6MTU0KQoJYXQgYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbi5ydW4oQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhOjM5KQoJYXQgYWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2suZXhlYyhBYnN0cmFjdERpc3BhdGNoZXIuc2NhbGE6NDE1KQoJYXQgc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2suZG9FeGVjKEZvcmtKb2luVGFzay5qYXZhOjI2MCkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29sJFdvcmtRdWV1ZS5ydW5UYXNrKEZvcmtKb2luUG9vbC5qYXZhOjEzMzkpCglhdCBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbC5ydW5Xb3JrZXIoRm9ya0pvaW5Qb29sLmphdmE6MTk3OSkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Xb3JrZXJUaHJlYWQucnVuKEZvcmtKb2luV29ya2VyVGhyZWFkLmphdmE6MTA3KQp0ACVqYXZhLnV0aWwuY29uY3VycmVudC5UaW1lb3V0RXhjZXB0aW9udXIAAltCrPMX+AYIVOACAAB4cAAABvas7QAFc3IAJWphdmEudXRpbC5jb25jdXJyZW50LlRpbWVvdXRFeGNlcHRpb24aYXK92+T1agIAAHhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdldAASTGphdmEvbGFuZy9TdHJpbmc7WwAKc3RhY2tUcmFjZXQAHltMamF2YS9sYW5nL1N0YWNrVHJhY2VFbGVtZW50O0wAFHN1cHByZXNzZWRFeGNlcHRpb25zdAAQTGphdmEvdXRpbC9MaXN0O3hwcQB+AAd0AExIZWFydGJlYXQgb2YgVGFza01hbmFnZXIgd2l0aCBpZCAwYTRlYTY1MTI0NDk4MmVmNGI0YjcwOTJkMThkZTc3NiB0aW1lZCBvdXQudXIAHltMamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50OwJGKjw8/SI5AgAAeHAAAAALc3IAG2phdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudGEJxZomNt2FAgAESQAKbGluZU51bWJlckwADmRlY2xhcmluZ0NsYXNzcQB+AARMAAhmaWxlTmFtZXEAfgAETAAKbWV0aG9kTmFtZXEAfgAEeHAAAAZ4dABJb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lcnQADkpvYk1hc3Rlci5qYXZhdAAWbm90aWZ5SGVhcnRiZWF0VGltZW91dHNxAH4ACwAAAVN0AEhvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3J0ABlIZWFydGJlYXRNYW5hZ2VySW1wbC5qYXZhdAADcnVuc3EAfgALAAAB/3QALmphdmEudXRpbC5jb25jdXJyZW50LkV4ZWN1dG9ycyRSdW5uYWJsZUFkYXB0ZXJ0AA5FeGVjdXRvcnMuamF2YXQABGNhbGxzcQB+AAsAAAEKdAAfamF2YS51dGlsLmNvbmN1cnJlbnQuRnV0dXJlVGFza3QAD0Z1dHVyZVRhc2suamF2YXEAfgATc3EAfgALAAAAmnQAYG9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5jb25jdXJyZW50LmFra2EuQWN0b3JTeXN0ZW1TY2hlZHVsZWRFeGVjdXRvckFkYXB0ZXIkU2NoZWR1bGVkRnV0dXJlVGFza3QAKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmFxAH4AE3NxAH4ACwAAACd0ABxha2thLmRpc3BhdGNoLlRhc2tJbnZvY2F0aW9udAAYQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhcQB+ABNzcQB+AAsAAAGfdAA7YWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2txAH4AIHQABGV4ZWNzcQB+AAsAAAEEdAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2t0ABFGb3JrSm9pblRhc2suamF2YXQABmRvRXhlY3NxAH4ACwAABTt0ADBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbCRXb3JrUXVldWV0ABFGb3JrSm9pblBvb2wuamF2YXQAB3J1blRhc2tzcQB+AAsAAAe7dAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2xxAH4AKnQACXJ1bldvcmtlcnNxAH4ACwAAAGt0AC5zY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luV29ya2VyVGhyZWFkdAAZRm9ya0pvaW5Xb3JrZXJUaHJlYWQuamF2YXEAfgATc3IAJmphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVMaXN0/A8lMbXsjhACAAFMAARsaXN0cQB+AAZ4cgAsamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZUNvbGxlY3Rpb24ZQgCAy173HgIAAUwAAWN0ABZMamF2YS91dGlsL0NvbGxlY3Rpb247eHBzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAAHcEAAAAAHhxAH4AN3g="}}}, httpResponseStatus=200 OK}
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "status" (class org.apache.flink.runtime.rest.messages.ErrorResponseBody), not marked as ignorable (one known property: "errors"])
at [Source: N/A; line: -1, column: -1] (through reference chain: org.apache.flink.runtime.rest.messages.ErrorResponseBody["status"])
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:851) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1085) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1392) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperties(BeanDeserializerBase.java:1346) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:455) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1127) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:298) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper._readValue(ObjectMapper.java:3779) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2050) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.treeToValue(ObjectMapper.java:2547) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
at org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:388) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:373) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:952) [?:1.8.0_192]
at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) [?:1.8.0_192]
at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) [?:1.8.0_192]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_192]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_192]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_192]
Re: flink java.util.concurrent.TimeoutException
Posted by zhisheng <zh...@gmail.com>.
检查一下这个 TM 的 GC 次数和时间吧
Yangze Guo <ka...@gmail.com> 于2020年4月15日周三 下午3:03写道：
> 日志上看是Taskmanager心跳超时了，如果tm还在，是不是网络问题呢？尝试把heartbeat.timeout调大一些试试？
>
> Best,
> Yangze Guo
>
> On Mon, Apr 13, 2020 at 10:40 AM 欧阳苗 <a9...@qq.com> wrote:
> >
> >
> job运行了两天就挂了，然后抛出如下异常，但是taskManager没有挂，其他的job还能正常在上面跑,请问这个问题是什么原因导致的，有什么好的解决办法吗
> >
> >
> > 2020-04-13 06:20:31.379 ERROR 1 --- [ent-IO-thread-3]
> org.apache.flink.runtime.rest.RestClient.parseResponse:393 : Received
> response was neither of the expected type ([simple type, class
> org.apache.flink.runtime.rest.messages.job.JobExecutionResultResponseBody])
> nor an error.
> Response=JsonResponse{json={"status":{"id":"COMPLETED"},"job-execution-result":{"id":"2d2a0b4efc8c3d973e2e9490b7b3b2f1","application-status":"FAILED","accumulator-results":{},"net-runtime":217272900,"failure-cause":{"class":"java.util.concurrent.TimeoutException","stack-trace":"java.util.concurrent.TimeoutException:
> Heartbeat of TaskManager with id 0a4ea651244982ef4b4b7092d18de776 timed
> out.\n\tat
> org.apache.flink.runtime.jobmaster.JobMaster$TaskManagerHeartbeatListener.notifyHeartbeatTimeout(JobMaster.java:1656)\n\tat
> org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl$HeartbeatMonitor.run(HeartbeatManagerImpl.java:339)\n\tat
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat
> java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat
> org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154)\n\tat
> akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)\n\tat
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:415)\n\tat
> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n","serialized-throwable":"rO0ABXNyAClvcmcuYXBhY2hlLmZsaW5rLnV0aWwuU2VyaWFsaXplZFRocm93YWJsZWUWnfUfpxPzAgADTAAZZnVsbFN0cmluZ2lmaWVkU3RhY2tUcmFjZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAFm9yaWdpbmFsRXJyb3JDbGFzc05hbWVxAH4AAVsAE3NlcmlhbGl6ZWRFeGNlcHRpb250AAJbQnhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdlcQB+AAFbAApzdGFja1RyYWNldAAeW0xqYXZhL2xhbmcvU3RhY2tUcmFjZUVsZW1lbnQ7TAAUc3VwcHJlc3NlZEV4Y2VwdGlvbnN0ABBMamF2YS91dGlsL0xpc3Q7eHBwdABMSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LnVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAC3NyABtqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnRhCcWaJjbdhQIABEkACmxpbmVOdW1iZXJMAA5kZWNsYXJpbmdDbGFzc3EAfgABTAAIZmlsZU5hbWVxAH4AAUwACm1ldGhvZE5hbWVxAH4AAXhwAAAGeHQASW9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5qb2JtYXN0ZXIuSm9iTWFzdGVyJFRhc2tNYW5hZ2VySGVhcnRiZWF0TGlzdGVuZXJ0AA5Kb2JNYXN0ZXIuamF2YXQAFm5vdGlmeUhlYXJ0YmVhdFRpbWVvdXRzcQB+AAwAAAFTdABIb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmhlYXJ0YmVhdC5IZWFydGJlYXRNYW5hZ2VySW1wbCRIZWFydGJlYXRNb25pdG9ydAAZSGVhcnRiZWF0TWFuYWdlckltcGwuamF2YXQAA3J1bnNxAH4ADAAAAf90AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmF0AARjYWxsc3EAfgAMAAABCnQAH2phdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2t0AA9GdXR1cmVUYXNrLmphdmFxAH4AFHNxAH4ADAAAAJp0AGBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2t0AChBY3RvclN5c3RlbVNjaGVkdWxlZEV4ZWN1dG9yQWRhcHRlci5qYXZhcQB+ABRzcQB+AAwAAAAndAAcYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbnQAGEFic3RyYWN0RGlzcGF0Y2hlci5zY2FsYXEAfgAUc3EAfgAMAAABn3QAO2Fra2EuZGlzcGF0Y2guRm9ya0pvaW5FeGVjdXRvckNvbmZpZ3VyYXRvciRBa2thRm9ya0pvaW5UYXNrcQB+ACF0AARleGVjc3EAfgAMAAABBHQAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5UYXNrdAARRm9ya0pvaW5UYXNrLmphdmF0AAZkb0V4ZWNzcQB+AAwAAAU7dAAwc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2wkV29ya1F1ZXVldAARRm9ya0pvaW5Qb29sLmphdmF0AAdydW5UYXNrc3EAfgAMAAAHu3QAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29scQB+ACt0AAlydW5Xb3JrZXJzcQB+AAwAAABrdAAuc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pbldvcmtlclRocmVhZHQAGUZvcmtKb2luV29ya2VyVGhyZWFkLmphdmFxAH4AFHNyACZqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTGlzdPwPJTG17I4QAgABTAAEbGlzdHEAfgAHeHIALGphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVDb2xsZWN0aW9uGUIAgMte9x4CAAFMAAFjdAAWTGphdmEvdXRpbC9Db2xsZWN0aW9uO3hwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4cQB+ADh4dARkamF2YS51dGlsLmNvbmN1cnJlbnQuVGltZW91dEV4Y2VwdGlvbjogSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LgoJYXQgb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lci5ub3RpZnlIZWFydGJlYXRUaW1lb3V0KEpvYk1hc3Rlci5qYXZhOjE2NTYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3IucnVuKEhlYXJ0YmVhdE1hbmFnZXJJbXBsLmphdmE6MzM5KQoJYXQgamF2YS51dGlsLmNvbmN1cnJlbnQuRXhlY3V0b3JzJFJ1bm5hYmxlQWRhcHRlci5jYWxsKEV4ZWN1dG9ycy5qYXZhOjUxMSkKCWF0IGphdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2sucnVuKEZ1dHVyZVRhc2suamF2YToyNjYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2sucnVuKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmE6MTU0KQoJYXQgYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbi5ydW4oQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhOjM5KQoJYXQgYWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2suZXhlYyhBYnN0cmFjdERpc3BhdGNoZXIuc2NhbGE6NDE1KQoJYXQgc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2suZG9FeGVjKEZvcmtKb2luVGFzay5qYXZhOjI2MCkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29sJFdvcmtRdWV1ZS5ydW5UYXNrKEZvcmtKb2luUG9vbC5qYXZhOjEzMzkpCglhdCBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbC5ydW5Xb3JrZXIoRm9ya0pvaW5Qb29sLmphdmE6MTk3OSkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Xb3JrZXJUaHJlYWQucnVuKEZvcmtKb2luV29ya2VyVGhyZWFkLmphdmE6MTA3KQp0ACVqYXZhLnV0aWwuY29uY3VycmVudC5UaW1lb3V0RXhjZXB0aW9udXIAAltCrPMX+AYIVOACAAB4cAAABvas7QAFc3IAJWphdmEudXRpbC5jb25jdXJyZW50LlRpbWVvdXRFeGNlcHRpb24aYXK92+T1agIAAHhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdldAASTGphdmEvbGFuZy9TdHJpbmc7WwAKc3RhY2tUcmFjZXQAHltMamF2YS9sYW5nL1N0YWNrVHJhY2VFbGVtZW50O0wAFHN1cHByZXNzZWRFeGNlcHRpb25zdAAQTGphdmEvdXRpbC9MaXN0O3hwcQB+AAd0AExIZWFydGJlYXQgb2YgVGFza01hbmFnZXIgd2l0aCBpZCAwYTRlYTY1MTI0NDk4MmVmNGI0YjcwOTJkMThkZTc3NiB0aW1lZCBvdXQudXIAHltMamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50OwJGKjw8/SI5AgAAeHAAAAALc3IAG2phdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudGEJxZomNt2FAgAESQAKbGluZU51bWJlckwADmRlY2xhcmluZ0NsYXNzcQB+AARMAAhmaWxlTmFtZXEAfgAETAAKbWV0aG9kTmFtZXEAfgAEeHAAAAZ4dABJb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lcnQADkpvYk1hc3Rlci5qYXZhdAAWbm90aWZ5SGVhcnRiZWF0VGltZW91dHNxAH4ACwAAAVN0AEhvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3J0ABlIZWFydGJlYXRNYW5hZ2VySW1wbC5qYXZhdAADcnVuc3EAfgALAAAB/3QALmphdmEudXRpbC5jb25jdXJyZW50LkV4ZWN1dG9ycyRSdW5uYWJsZUFkYXB0ZXJ0AA5FeGVjdXRvcnMuamF2YXQABGNhbGxzcQB+AAsAAAEKdAAfamF2YS51dGlsLmNvbmN1cnJlbnQuRnV0dXJlVGFza3QAD0Z1dHVyZVRhc2suamF2YXEAfgATc3EAfgALAAAAmnQAYG9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5jb25jdXJyZW50LmFra2EuQWN0b3JTeXN0ZW1TY2hlZHVsZWRFeGVjdXRvckFkYXB0ZXIkU2NoZWR1bGVkRnV0dXJlVGFza3QAKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmFxAH4AE3NxAH4ACwAAACd0ABxha2thLmRpc3BhdGNoLlRhc2tJbnZvY2F0aW9udAAYQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhcQB+ABNzcQB+AAsAAAGfdAA7YWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2txAH4AIHQABGV4ZWNzcQB+AAsAAAEEdAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2t0ABFGb3JrSm9pblRhc2suamF2YXQABmRvRXhlY3NxAH4ACwAABTt0ADBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbCRXb3JrUXVldWV0ABFGb3JrSm9pblBvb2wuamF2YXQAB3J1blRhc2tzcQB+AAsAAAe7dAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2xxAH4AKnQACXJ1bldvcmtlcnNxAH4ACwAAAGt0AC5zY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luV29ya2VyVGhyZWFkdAAZRm9ya0pvaW5Xb3JrZXJUaHJlYWQuamF2YXEAfgATc3IAJmphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVMaXN0/A8lMbXsjhACAAFMAARsaXN0cQB+AAZ4cgAsamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZUNvbGxlY3Rpb24ZQgCAy173HgIAAUwAAWN0ABZMamF2YS91dGlsL0NvbGxlY3Rpb247eHBzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAAHcEAAAAAHhxAH4AN3g="}}},
> httpResponseStatus=200 OK}
> >
> >
> >
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException:
> Unrecognized field "status" (class
> org.apache.flink.runtime.rest.messages.ErrorResponseBody), not marked as
> ignorable (one known property: "errors"])
> > at [Source: N/A; line: -1, column: -1] (through reference chain:
> org.apache.flink.runtime.rest.messages.ErrorResponseBody["status"])
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:851)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1085)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1392)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperties(BeanDeserializerBase.java:1346)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:455)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1127)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:298)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper._readValue(ObjectMapper.java:3779)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2050)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.treeToValue(ObjectMapper.java:2547)
> ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> > at
> org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:388)
> ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> > at
> org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:373)
> ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> > at
> java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:952)
> [?:1.8.0_192]
> > at
> java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926)
> [?:1.8.0_192]
> > at
> java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442)
> [?:1.8.0_192]
> > at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [?:1.8.0_192]
> > at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [?:1.8.0_192]
> > at java.lang.Thread.run(Thread.java:748) [?:1.8.0_192]
>
Re: flink java.util.concurrent.TimeoutException
Posted by Yangze Guo <ka...@gmail.com>.
日志上看是Taskmanager心跳超时了，如果tm还在，是不是网络问题呢？尝试把heartbeat.timeout调大一些试试？
Best,
Yangze Guo
On Mon, Apr 13, 2020 at 10:40 AM 欧阳苗 <a9...@qq.com> wrote:
>
> job运行了两天就挂了，然后抛出如下异常，但是taskManager没有挂，其他的job还能正常在上面跑,请问这个问题是什么原因导致的，有什么好的解决办法吗
>
>
> 2020-04-13 06:20:31.379 ERROR 1 --- [ent-IO-thread-3] org.apache.flink.runtime.rest.RestClient.parseResponse:393 : Received response was neither of the expected type ([simple type, class org.apache.flink.runtime.rest.messages.job.JobExecutionResultResponseBody]) nor an error. Response=JsonResponse{json={"status":{"id":"COMPLETED"},"job-execution-result":{"id":"2d2a0b4efc8c3d973e2e9490b7b3b2f1","application-status":"FAILED","accumulator-results":{},"net-runtime":217272900,"failure-cause":{"class":"java.util.concurrent.TimeoutException","stack-trace":"java.util.concurrent.TimeoutException: Heartbeat of TaskManager with id 0a4ea651244982ef4b4b7092d18de776 timed out.\n\tat org.apache.flink.runtime.jobmaster.JobMaster$TaskManagerHeartbeatListener.notifyHeartbeatTimeout(JobMaster.java:1656)\n\tat org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl$HeartbeatMonitor.run(HeartbeatManagerImpl.java:339)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154)\n\tat akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)\n\tat akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:415)\n\tat scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n","serialized-throwable":"rO0ABXNyAClvcmcuYXBhY2hlLmZsaW5rLnV0aWwuU2VyaWFsaXplZFRocm93YWJsZWUWnfUfpxPzAgADTAAZZnVsbFN0cmluZ2lmaWVkU3RhY2tUcmFjZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAFm9yaWdpbmFsRXJyb3JDbGFzc05hbWVxAH4AAVsAE3NlcmlhbGl6ZWRFeGNlcHRpb250AAJbQnhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdlcQB+AAFbAApzdGFja1RyYWNldAAeW0xqYXZhL2xhbmcvU3RhY2tUcmFjZUVsZW1lbnQ7TAAUc3VwcHJlc3NlZEV4Y2VwdGlvbnN0ABBMamF2YS91dGlsL0xpc3Q7eHBwdABMSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LnVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAC3NyABtqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnRhCcWaJjbdhQIABEkACmxpbmVOdW1iZXJMAA5kZWNsYXJpbmdDbGFzc3EAfgABTAAIZmlsZU5hbWVxAH4AAUwACm1ldGhvZE5hbWVxAH4AAXhwAAAGeHQASW9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5qb2JtYXN0ZXIuSm9iTWFzdGVyJFRhc2tNYW5hZ2VySGVhcnRiZWF0TGlzdGVuZXJ0AA5Kb2JNYXN0ZXIuamF2YXQAFm5vdGlmeUhlYXJ0YmVhdFRpbWVvdXRzcQB+AAwAAAFTdABIb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmhlYXJ0YmVhdC5IZWFydGJlYXRNYW5hZ2VySW1wbCRIZWFydGJlYXRNb25pdG9ydAAZSGVhcnRiZWF0TWFuYWdlckltcGwuamF2YXQAA3J1bnNxAH4ADAAAAf90AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmF0AARjYWxsc3EAfgAMAAABCnQAH2phdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2t0AA9GdXR1cmVUYXNrLmphdmFxAH4AFHNxAH4ADAAAAJp0AGBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2t0AChBY3RvclN5c3RlbVNjaGVkdWxlZEV4ZWN1dG9yQWRhcHRlci5qYXZhcQB+ABRzcQB+AAwAAAAndAAcYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbnQAGEFic3RyYWN0RGlzcGF0Y2hlci5zY2FsYXEAfgAUc3EAfgAMAAABn3QAO2Fra2EuZGlzcGF0Y2guRm9ya0pvaW5FeGVjdXRvckNvbmZpZ3VyYXRvciRBa2thRm9ya0pvaW5UYXNrcQB+ACF0AARleGVjc3EAfgAMAAABBHQAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5UYXNrdAARRm9ya0pvaW5UYXNrLmphdmF0AAZkb0V4ZWNzcQB+AAwAAAU7dAAwc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2wkV29ya1F1ZXVldAARRm9ya0pvaW5Qb29sLmphdmF0AAdydW5UYXNrc3EAfgAMAAAHu3QAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29scQB+ACt0AAlydW5Xb3JrZXJzcQB+AAwAAABrdAAuc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pbldvcmtlclRocmVhZHQAGUZvcmtKb2luV29ya2VyVGhyZWFkLmphdmFxAH4AFHNyACZqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTGlzdPwPJTG17I4QAgABTAAEbGlzdHEAfgAHeHIALGphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVDb2xsZWN0aW9uGUIAgMte9x4CAAFMAAFjdAAWTGphdmEvdXRpbC9Db2xsZWN0aW9uO3hwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4cQB+ADh4dARkamF2YS51dGlsLmNvbmN1cnJlbnQuVGltZW91dEV4Y2VwdGlvbjogSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LgoJYXQgb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lci5ub3RpZnlIZWFydGJlYXRUaW1lb3V0KEpvYk1hc3Rlci5qYXZhOjE2NTYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3IucnVuKEhlYXJ0YmVhdE1hbmFnZXJJbXBsLmphdmE6MzM5KQoJYXQgamF2YS51dGlsLmNvbmN1cnJlbnQuRXhlY3V0b3JzJFJ1bm5hYmxlQWRhcHRlci5jYWxsKEV4ZWN1dG9ycy5qYXZhOjUxMSkKCWF0IGphdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2sucnVuKEZ1dHVyZVRhc2suamF2YToyNjYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2sucnVuKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmE6MTU0KQoJYXQgYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbi5ydW4oQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhOjM5KQoJYXQgYWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2suZXhlYyhBYnN0cmFjdERpc3BhdGNoZXIuc2NhbGE6NDE1KQoJYXQgc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2suZG9FeGVjKEZvcmtKb2luVGFzay5qYXZhOjI2MCkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29sJFdvcmtRdWV1ZS5ydW5UYXNrKEZvcmtKb2luUG9vbC5qYXZhOjEzMzkpCglhdCBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbC5ydW5Xb3JrZXIoRm9ya0pvaW5Qb29sLmphdmE6MTk3OSkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Xb3JrZXJUaHJlYWQucnVuKEZvcmtKb2luV29ya2VyVGhyZWFkLmphdmE6MTA3KQp0ACVqYXZhLnV0aWwuY29uY3VycmVudC5UaW1lb3V0RXhjZXB0aW9udXIAAltCrPMX+AYIVOACAAB4cAAABvas7QAFc3IAJWphdmEudXRpbC5jb25jdXJyZW50LlRpbWVvdXRFeGNlcHRpb24aYXK92+T1agIAAHhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdldAASTGphdmEvbGFuZy9TdHJpbmc7WwAKc3RhY2tUcmFjZXQAHltMamF2YS9sYW5nL1N0YWNrVHJhY2VFbGVtZW50O0wAFHN1cHByZXNzZWRFeGNlcHRpb25zdAAQTGphdmEvdXRpbC9MaXN0O3hwcQB+AAd0AExIZWFydGJlYXQgb2YgVGFza01hbmFnZXIgd2l0aCBpZCAwYTRlYTY1MTI0NDk4MmVmNGI0YjcwOTJkMThkZTc3NiB0aW1lZCBvdXQudXIAHltMamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50OwJGKjw8/SI5AgAAeHAAAAALc3IAG2phdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudGEJxZomNt2FAgAESQAKbGluZU51bWJlckwADmRlY2xhcmluZ0NsYXNzcQB+AARMAAhmaWxlTmFtZXEAfgAETAAKbWV0aG9kTmFtZXEAfgAEeHAAAAZ4dABJb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lcnQADkpvYk1hc3Rlci5qYXZhdAAWbm90aWZ5SGVhcnRiZWF0VGltZW91dHNxAH4ACwAAAVN0AEhvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3J0ABlIZWFydGJlYXRNYW5hZ2VySW1wbC5qYXZhdAADcnVuc3EAfgALAAAB/3QALmphdmEudXRpbC5jb25jdXJyZW50LkV4ZWN1dG9ycyRSdW5uYWJsZUFkYXB0ZXJ0AA5FeGVjdXRvcnMuamF2YXQABGNhbGxzcQB+AAsAAAEKdAAfamF2YS51dGlsLmNvbmN1cnJlbnQuRnV0dXJlVGFza3QAD0Z1dHVyZVRhc2suamF2YXEAfgATc3EAfgALAAAAmnQAYG9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5jb25jdXJyZW50LmFra2EuQWN0b3JTeXN0ZW1TY2hlZHVsZWRFeGVjdXRvckFkYXB0ZXIkU2NoZWR1bGVkRnV0dXJlVGFza3QAKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmFxAH4AE3NxAH4ACwAAACd0ABxha2thLmRpc3BhdGNoLlRhc2tJbnZvY2F0aW9udAAYQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhcQB+ABNzcQB+AAsAAAGfdAA7YWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2txAH4AIHQABGV4ZWNzcQB+AAsAAAEEdAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2t0ABFGb3JrSm9pblRhc2suamF2YXQABmRvRXhlY3NxAH4ACwAABTt0ADBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbCRXb3JrUXVldWV0ABFGb3JrSm9pblBvb2wuamF2YXQAB3J1blRhc2tzcQB+AAsAAAe7dAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2xxAH4AKnQACXJ1bldvcmtlcnNxAH4ACwAAAGt0AC5zY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luV29ya2VyVGhyZWFkdAAZRm9ya0pvaW5Xb3JrZXJUaHJlYWQuamF2YXEAfgATc3IAJmphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVMaXN0/A8lMbXsjhACAAFMAARsaXN0cQB+AAZ4cgAsamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZUNvbGxlY3Rpb24ZQgCAy173HgIAAUwAAWN0ABZMamF2YS91dGlsL0NvbGxlY3Rpb247eHBzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAAHcEAAAAAHhxAH4AN3g="}}}, httpResponseStatus=200 OK}
>
>
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "status" (class org.apache.flink.runtime.rest.messages.ErrorResponseBody), not marked as ignorable (one known property: "errors"])
> at [Source: N/A; line: -1, column: -1] (through reference chain: org.apache.flink.runtime.rest.messages.ErrorResponseBody["status"])
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:851) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1085) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1392) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperties(BeanDeserializerBase.java:1346) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:455) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1127) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:298) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper._readValue(ObjectMapper.java:3779) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2050) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.treeToValue(ObjectMapper.java:2547) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:388) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:373) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:952) [?:1.8.0_192]
> at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) [?:1.8.0_192]
> at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) [?:1.8.0_192]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_192]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_192]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_192]
Re: flink java.util.concurrent.TimeoutException
Posted by Yangze Guo <ka...@gmail.com>.
日志上看是Taskmanager心跳超时了，如果tm还在，是不是网络问题呢？尝试把heartbeat.timeout调大一些试试？
Best,
Yangze Guo
On Mon, Apr 13, 2020 at 10:40 AM 欧阳苗 <a9...@qq.com> wrote:
>
> job运行了两天就挂了，然后抛出如下异常，但是taskManager没有挂，其他的job还能正常在上面跑,请问这个问题是什么原因导致的，有什么好的解决办法吗
>
>
> 2020-04-13 06:20:31.379 ERROR 1 --- [ent-IO-thread-3] org.apache.flink.runtime.rest.RestClient.parseResponse:393 : Received response was neither of the expected type ([simple type, class org.apache.flink.runtime.rest.messages.job.JobExecutionResultResponseBody]) nor an error. Response=JsonResponse{json={"status":{"id":"COMPLETED"},"job-execution-result":{"id":"2d2a0b4efc8c3d973e2e9490b7b3b2f1","application-status":"FAILED","accumulator-results":{},"net-runtime":217272900,"failure-cause":{"class":"java.util.concurrent.TimeoutException","stack-trace":"java.util.concurrent.TimeoutException: Heartbeat of TaskManager with id 0a4ea651244982ef4b4b7092d18de776 timed out.\n\tat org.apache.flink.runtime.jobmaster.JobMaster$TaskManagerHeartbeatListener.notifyHeartbeatTimeout(JobMaster.java:1656)\n\tat org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl$HeartbeatMonitor.run(HeartbeatManagerImpl.java:339)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154)\n\tat akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)\n\tat akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:415)\n\tat scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n","serialized-throwable":"rO0ABXNyAClvcmcuYXBhY2hlLmZsaW5rLnV0aWwuU2VyaWFsaXplZFRocm93YWJsZWUWnfUfpxPzAgADTAAZZnVsbFN0cmluZ2lmaWVkU3RhY2tUcmFjZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAFm9yaWdpbmFsRXJyb3JDbGFzc05hbWVxAH4AAVsAE3NlcmlhbGl6ZWRFeGNlcHRpb250AAJbQnhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdlcQB+AAFbAApzdGFja1RyYWNldAAeW0xqYXZhL2xhbmcvU3RhY2tUcmFjZUVsZW1lbnQ7TAAUc3VwcHJlc3NlZEV4Y2VwdGlvbnN0ABBMamF2YS91dGlsL0xpc3Q7eHBwdABMSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LnVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAC3NyABtqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnRhCcWaJjbdhQIABEkACmxpbmVOdW1iZXJMAA5kZWNsYXJpbmdDbGFzc3EAfgABTAAIZmlsZU5hbWVxAH4AAUwACm1ldGhvZE5hbWVxAH4AAXhwAAAGeHQASW9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5qb2JtYXN0ZXIuSm9iTWFzdGVyJFRhc2tNYW5hZ2VySGVhcnRiZWF0TGlzdGVuZXJ0AA5Kb2JNYXN0ZXIuamF2YXQAFm5vdGlmeUhlYXJ0YmVhdFRpbWVvdXRzcQB+AAwAAAFTdABIb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmhlYXJ0YmVhdC5IZWFydGJlYXRNYW5hZ2VySW1wbCRIZWFydGJlYXRNb25pdG9ydAAZSGVhcnRiZWF0TWFuYWdlckltcGwuamF2YXQAA3J1bnNxAH4ADAAAAf90AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmF0AARjYWxsc3EAfgAMAAABCnQAH2phdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2t0AA9GdXR1cmVUYXNrLmphdmFxAH4AFHNxAH4ADAAAAJp0AGBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2t0AChBY3RvclN5c3RlbVNjaGVkdWxlZEV4ZWN1dG9yQWRhcHRlci5qYXZhcQB+ABRzcQB+AAwAAAAndAAcYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbnQAGEFic3RyYWN0RGlzcGF0Y2hlci5zY2FsYXEAfgAUc3EAfgAMAAABn3QAO2Fra2EuZGlzcGF0Y2guRm9ya0pvaW5FeGVjdXRvckNvbmZpZ3VyYXRvciRBa2thRm9ya0pvaW5UYXNrcQB+ACF0AARleGVjc3EAfgAMAAABBHQAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5UYXNrdAARRm9ya0pvaW5UYXNrLmphdmF0AAZkb0V4ZWNzcQB+AAwAAAU7dAAwc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2wkV29ya1F1ZXVldAARRm9ya0pvaW5Qb29sLmphdmF0AAdydW5UYXNrc3EAfgAMAAAHu3QAJnNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29scQB+ACt0AAlydW5Xb3JrZXJzcQB+AAwAAABrdAAuc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pbldvcmtlclRocmVhZHQAGUZvcmtKb2luV29ya2VyVGhyZWFkLmphdmFxAH4AFHNyACZqYXZhLnV0aWwuQ29sbGVjdGlvbnMkVW5tb2RpZmlhYmxlTGlzdPwPJTG17I4QAgABTAAEbGlzdHEAfgAHeHIALGphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVDb2xsZWN0aW9uGUIAgMte9x4CAAFMAAFjdAAWTGphdmEvdXRpbC9Db2xsZWN0aW9uO3hwc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4cQB+ADh4dARkamF2YS51dGlsLmNvbmN1cnJlbnQuVGltZW91dEV4Y2VwdGlvbjogSGVhcnRiZWF0IG9mIFRhc2tNYW5hZ2VyIHdpdGggaWQgMGE0ZWE2NTEyNDQ5ODJlZjRiNGI3MDkyZDE4ZGU3NzYgdGltZWQgb3V0LgoJYXQgb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lci5ub3RpZnlIZWFydGJlYXRUaW1lb3V0KEpvYk1hc3Rlci5qYXZhOjE2NTYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3IucnVuKEhlYXJ0YmVhdE1hbmFnZXJJbXBsLmphdmE6MzM5KQoJYXQgamF2YS51dGlsLmNvbmN1cnJlbnQuRXhlY3V0b3JzJFJ1bm5hYmxlQWRhcHRlci5jYWxsKEV4ZWN1dG9ycy5qYXZhOjUxMSkKCWF0IGphdmEudXRpbC5jb25jdXJyZW50LkZ1dHVyZVRhc2sucnVuKEZ1dHVyZVRhc2suamF2YToyNjYpCglhdCBvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuY29uY3VycmVudC5ha2thLkFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyJFNjaGVkdWxlZEZ1dHVyZVRhc2sucnVuKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmE6MTU0KQoJYXQgYWtrYS5kaXNwYXRjaC5UYXNrSW52b2NhdGlvbi5ydW4oQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhOjM5KQoJYXQgYWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2suZXhlYyhBYnN0cmFjdERpc3BhdGNoZXIuc2NhbGE6NDE1KQoJYXQgc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2suZG9FeGVjKEZvcmtKb2luVGFzay5qYXZhOjI2MCkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Qb29sJFdvcmtRdWV1ZS5ydW5UYXNrKEZvcmtKb2luUG9vbC5qYXZhOjEzMzkpCglhdCBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbC5ydW5Xb3JrZXIoRm9ya0pvaW5Qb29sLmphdmE6MTk3OSkKCWF0IHNjYWxhLmNvbmN1cnJlbnQuZm9ya2pvaW4uRm9ya0pvaW5Xb3JrZXJUaHJlYWQucnVuKEZvcmtKb2luV29ya2VyVGhyZWFkLmphdmE6MTA3KQp0ACVqYXZhLnV0aWwuY29uY3VycmVudC5UaW1lb3V0RXhjZXB0aW9udXIAAltCrPMX+AYIVOACAAB4cAAABvas7QAFc3IAJWphdmEudXRpbC5jb25jdXJyZW50LlRpbWVvdXRFeGNlcHRpb24aYXK92+T1agIAAHhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdldAASTGphdmEvbGFuZy9TdHJpbmc7WwAKc3RhY2tUcmFjZXQAHltMamF2YS9sYW5nL1N0YWNrVHJhY2VFbGVtZW50O0wAFHN1cHByZXNzZWRFeGNlcHRpb25zdAAQTGphdmEvdXRpbC9MaXN0O3hwcQB+AAd0AExIZWFydGJlYXQgb2YgVGFza01hbmFnZXIgd2l0aCBpZCAwYTRlYTY1MTI0NDk4MmVmNGI0YjcwOTJkMThkZTc3NiB0aW1lZCBvdXQudXIAHltMamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50OwJGKjw8/SI5AgAAeHAAAAALc3IAG2phdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudGEJxZomNt2FAgAESQAKbGluZU51bWJlckwADmRlY2xhcmluZ0NsYXNzcQB+AARMAAhmaWxlTmFtZXEAfgAETAAKbWV0aG9kTmFtZXEAfgAEeHAAAAZ4dABJb3JnLmFwYWNoZS5mbGluay5ydW50aW1lLmpvYm1hc3Rlci5Kb2JNYXN0ZXIkVGFza01hbmFnZXJIZWFydGJlYXRMaXN0ZW5lcnQADkpvYk1hc3Rlci5qYXZhdAAWbm90aWZ5SGVhcnRiZWF0VGltZW91dHNxAH4ACwAAAVN0AEhvcmcuYXBhY2hlLmZsaW5rLnJ1bnRpbWUuaGVhcnRiZWF0LkhlYXJ0YmVhdE1hbmFnZXJJbXBsJEhlYXJ0YmVhdE1vbml0b3J0ABlIZWFydGJlYXRNYW5hZ2VySW1wbC5qYXZhdAADcnVuc3EAfgALAAAB/3QALmphdmEudXRpbC5jb25jdXJyZW50LkV4ZWN1dG9ycyRSdW5uYWJsZUFkYXB0ZXJ0AA5FeGVjdXRvcnMuamF2YXQABGNhbGxzcQB+AAsAAAEKdAAfamF2YS51dGlsLmNvbmN1cnJlbnQuRnV0dXJlVGFza3QAD0Z1dHVyZVRhc2suamF2YXEAfgATc3EAfgALAAAAmnQAYG9yZy5hcGFjaGUuZmxpbmsucnVudGltZS5jb25jdXJyZW50LmFra2EuQWN0b3JTeXN0ZW1TY2hlZHVsZWRFeGVjdXRvckFkYXB0ZXIkU2NoZWR1bGVkRnV0dXJlVGFza3QAKEFjdG9yU3lzdGVtU2NoZWR1bGVkRXhlY3V0b3JBZGFwdGVyLmphdmFxAH4AE3NxAH4ACwAAACd0ABxha2thLmRpc3BhdGNoLlRhc2tJbnZvY2F0aW9udAAYQWJzdHJhY3REaXNwYXRjaGVyLnNjYWxhcQB+ABNzcQB+AAsAAAGfdAA7YWtrYS5kaXNwYXRjaC5Gb3JrSm9pbkV4ZWN1dG9yQ29uZmlndXJhdG9yJEFra2FGb3JrSm9pblRhc2txAH4AIHQABGV4ZWNzcQB+AAsAAAEEdAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblRhc2t0ABFGb3JrSm9pblRhc2suamF2YXQABmRvRXhlY3NxAH4ACwAABTt0ADBzY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luUG9vbCRXb3JrUXVldWV0ABFGb3JrSm9pblBvb2wuamF2YXQAB3J1blRhc2tzcQB+AAsAAAe7dAAmc2NhbGEuY29uY3VycmVudC5mb3Jram9pbi5Gb3JrSm9pblBvb2xxAH4AKnQACXJ1bldvcmtlcnNxAH4ACwAAAGt0AC5zY2FsYS5jb25jdXJyZW50LmZvcmtqb2luLkZvcmtKb2luV29ya2VyVGhyZWFkdAAZRm9ya0pvaW5Xb3JrZXJUaHJlYWQuamF2YXEAfgATc3IAJmphdmEudXRpbC5Db2xsZWN0aW9ucyRVbm1vZGlmaWFibGVMaXN0/A8lMbXsjhACAAFMAARsaXN0cQB+AAZ4cgAsamF2YS51dGlsLkNvbGxlY3Rpb25zJFVubW9kaWZpYWJsZUNvbGxlY3Rpb24ZQgCAy173HgIAAUwAAWN0ABZMamF2YS91dGlsL0NvbGxlY3Rpb247eHBzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAAHcEAAAAAHhxAH4AN3g="}}}, httpResponseStatus=200 OK}
>
>
> org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "status" (class org.apache.flink.runtime.rest.messages.ErrorResponseBody), not marked as ignorable (one known property: "errors"])
> at [Source: N/A; line: -1, column: -1] (through reference chain: org.apache.flink.runtime.rest.messages.ErrorResponseBody["status"])
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:851) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1085) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1392) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperties(BeanDeserializerBase.java:1346) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:455) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1127) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:298) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:133) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper._readValue(ObjectMapper.java:3779) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2050) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper.treeToValue(ObjectMapper.java:2547) ~[flink-shaded-jackson-2.7.9-6.0.jar!/:2.7.9-6.0]
> at org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:388) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:373) ~[flink-runtime_2.12-1.8.0.jar!/:1.8.0]
> at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:952) [?:1.8.0_192]
> at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) [?:1.8.0_192]
> at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) [?:1.8.0_192]
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_192]
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_192]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_192]