You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by 陈帅 <ca...@126.com> on 2021/01/02 01:42:01 UTC
flink 1.11.2 非原生k8s session模式作业运行失败
我在尝试运行flink on 非原生 k8s session cluster时遇到示例作业运行失败的情况，还请解答一下，谢谢！
参考了官网给的k8s yaml配置文件 [1]，k8s session cluster能够运行起来，flink:latest下载的是v1.11.2版本的flink
按照其配置起了2个TM，每个TM开了一个slot，一共是两个slots。
之后我通过web UI提交了一个自带的TopSpeedWindowing示例，结果作业在scheduling一段时间后直接fail了，通过kubectl logs查看JM日志如下，TM没有输出日志
ExecutingTopSpeedWindowing example with default input data set.
Use --input to specify file input.
Printing result to stdout. Use --output to specify output path.
15:18:06.092 [flink-rest-server-netty-worker-thread-1] ERROR org.apache.flink.runtime.rest.handler.job.metrics.JobVertexWatermarksHandler - Unhandledexception.
java.util.concurrent.CancellationException: null
at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276) ~[?:1.8.0_275]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:99) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraph(DefaultExecutionGraphCache.java:71) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:75) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:73) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:178) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:81) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_275]
at org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:46) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:78) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:110) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:89) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:54) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:174) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:68) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:328) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:302) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1421) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:697) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:632) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:549) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [flink-dist_2.12-1.11.2.jar:1.11.2]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
15:18:06.147 [flink-rest-server-netty-worker-thread-1] ERROR org.apache.flink.runtime.rest.handler.job.metrics.JobVertexWatermarksHandler - Unhandledexception.
java.util.concurrent.CancellationException: null
at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276) ~[?:1.8.0_275]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:99) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraph(DefaultExecutionGraphCache.java:71) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:75) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:73) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:178) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:81) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_275]
at org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:46) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:78) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:110) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:89) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:54) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:174) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:68) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:328) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:302) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1421) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:697) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:632) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:549) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918) [flink-dist_2.12-1.11.2.jar:1.11.2]
at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [flink-dist_2.12-1.11.2.jar:1.11.2]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
15:22:38.352 [flink-akka.actor.default-dispatcher-29] ERROR org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogFileHandler - Failed to transfer file fromTaskExecutor08925453a916dc56068f47cd4c3f5967.
java.util.concurrent.CompletionException: org.apache.flink.util.FlinkException: The file LOG does not exist on the TaskExecutor.
at org.apache.flink.runtime.taskexecutor.TaskExecutor.lambda$requestFileUploadByFilePath$25(TaskExecutor.java:1742) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Causedby: org.apache.flink.util.FlinkException: The file LOG does not exist on the TaskExecutor.
... 5 more
15:22:38.362 [flink-akka.actor.default-dispatcher-29] ERROR org.apache.flink.runtime.rest.handler.taskmanager.TaskManagerLogFileHandler - Unhandledexception.
org.apache.flink.util.FlinkException: The file LOG does not exist on the TaskExecutor.
at org.apache.flink.runtime.taskexecutor.TaskExecutor.lambda$requestFileUploadByFilePath$25(TaskExecutor.java:1742) ~[flink-dist_2.12-1.11.2.jar:1.11.2]
at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
[1] https://ci.apache.org/projects/flink/flink-docs-release-1.10/ops/deployment/kubernetes.html#session-cluster-resource-definitions