You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Sebastian Struss <st...@justtrack.io> on 2022/03/25 08:42:27 UTC
Flink Web ui not stable in kubernetes?
Hello all,
i've been setting up flink in my kubernetes cluster with 2 job managers and
1 task manager (custom helm chart i wrote, no flink CLI used).
I can access the web ui, but often it seems to switch pods which i am
connected to and as soon as i am connected to the standby job manager it
doesn't load at all.
The leader election does seem to work nicely, as when i kill the leading
pod the standby instance takes over after ~5s.
I do see errors like this when i browse the web ui:
"""
2022-03-24 15:38:17,269 ERROR
org.apache.flink.runtime.rest.handler.job.JobDetailsHandler
[] - Unhandled exception.
java.util.concurrent.CancellationException: null
at
java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276)
~[?:1.8.0_302]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:98)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInfo(DefaultExecutionGraphCache.java:67)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:81)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:83)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:195)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:83)
~[flink-dist_2.12-1.14.2.jar:1.14.2]
at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_302]
at org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:45)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:80)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:115)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:94)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:55)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:238)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:71)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
[flink-dist_2.12-1.14.2.jar:1.14.2]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302]
"""
What can i do to track this down?
best regards,
Sebastian
Re: Flink Web ui not stable in kubernetes?
Posted by Guillaume Vauvert <gu...@gmail.com>.
Hi,
I agree that the changelog description of
https://issues.apache.org/jira/browse/FLINK-25732 is talking only about
technical root cause, not user consequences.
I have added a comment in https://issues.apache.org/jira/browse/FLINK-25732.
Have a nice day !
Guillaume
On 25/03/2022 10.31, Sebastian Struss wrote:
> Hi Guillaume,
>
> thank you for this great hint! It indeed fixed the mentioned issue.
> Just from reading the changelog of 1.14.4 i would not have known that
> this fix is included, maybe i was searching for the wrong stuff though.
>
> Have a great day!
> Sebastian
>
> On Fri, Mar 25, 2022 at 10:51 AM Guillaume Vauvert
> <gu...@gmail.com> wrote:
>
> Hello Sebastian,
>
> Multiple versions of Flink 1.14.x are known to have issue with
> UI/CLI, please switch to Flink 1.14.4.
>
> Best regards,
>
> Guillaume
>
> On 25/03/2022 08.42, Sebastian Struss wrote:
>> Hello all,
>>
>> i've been setting up flink in my kubernetes cluster with 2 job
>> managers and 1 task manager (custom helm chart i wrote, no
>> flink CLI used).
>> I can access the web ui, but often it seems to switch pods which
>> i am connected to and as soon as i am connected to the standby
>> job manager it doesn't load at all.
>> The leader election does seem to work nicely, as when i kill the
>> leading pod the standby instance takes over after ~5s.
>> I do see errors like this when i browse the web ui:
>>
>> """
>> 2022-03-24 15:38:17,269 ERROR
>> org.apache.flink.runtime.rest.handler.job.JobDetailsHandler [] -
>> Unhandled exception.
>> java.util.concurrent.CancellationException: null
>> at
>> java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276)
>> ~[?:1.8.0_302]
>> at
>> org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:98)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInfo(DefaultExecutionGraphCache.java:67)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:81)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:83)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:195)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:83)
>> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
>> at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_302]
>> at
>> org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:45)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:80)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:115)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:94)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:55)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:238)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:71)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at
>> org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
>> [flink-dist_2.12-1.14.2.jar:1.14.2]
>> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302]
>> """
>>
>> What can i do to track this down?
>>
>> best regards,
>> Sebastian
>
Re: Flink Web ui not stable in kubernetes?
Posted by Sebastian Struss <st...@justtrack.io>.
Hi Guillaume,
thank you for this great hint! It indeed fixed the mentioned issue.
Just from reading the changelog of 1.14.4 i would not have known that this
fix is included, maybe i was searching for the wrong stuff though.
Have a great day!
Sebastian
On Fri, Mar 25, 2022 at 10:51 AM Guillaume Vauvert <
guillaume.vauvert.pro@gmail.com> wrote:
> Hello Sebastian,
>
> Multiple versions of Flink 1.14.x are known to have issue with UI/CLI,
> please switch to Flink 1.14.4.
>
> Best regards,
>
> Guillaume
> On 25/03/2022 08.42, Sebastian Struss wrote:
>
> Hello all,
>
> i've been setting up flink in my kubernetes cluster with 2 job managers
> and 1 task manager (custom helm chart i wrote, no flink CLI used).
> I can access the web ui, but often it seems to switch pods which i am
> connected to and as soon as i am connected to the standby job manager it
> doesn't load at all.
> The leader election does seem to work nicely, as when i kill the leading
> pod the standby instance takes over after ~5s.
> I do see errors like this when i browse the web ui:
>
> """
> 2022-03-24 15:38:17,269 ERROR org.apache.flink.runtime.rest.handler.job.JobDetailsHandler
> [] - Unhandled exception.
> java.util.concurrent.CancellationException: null
> at
> java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276)
> ~[?:1.8.0_302]
> at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:98)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInfo(DefaultExecutionGraphCache.java:67)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:81)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:83)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:195)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:83)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_302]
> at org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:45)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:80)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:115)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:94)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:55)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:238)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:71)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302]
> """
>
> What can i do to track this down?
>
> best regards,
> Sebastian
>
>
Re: Flink Web ui not stable in kubernetes?
Posted by Guillaume Vauvert <gu...@gmail.com>.
Hello Sebastian,
Multiple versions of Flink 1.14.x are known to have issue with UI/CLI,
please switch to Flink 1.14.4.
Best regards,
Guillaume
On 25/03/2022 08.42, Sebastian Struss wrote:
> Hello all,
>
> i've been setting up flink in my kubernetes cluster with 2 job
> managers and 1 task manager (custom helm chart i wrote, no flink CLI
> used).
> I can access the web ui, but often it seems to switch pods which i am
> connected to and as soon as i am connected to the standby job manager
> it doesn't load at all.
> The leader election does seem to work nicely, as when i kill the
> leading pod the standby instance takes over after ~5s.
> I do see errors like this when i browse the web ui:
>
> """
> 2022-03-24 15:38:17,269 ERROR
> org.apache.flink.runtime.rest.handler.job.JobDetailsHandler [] -
> Unhandled exception.
> java.util.concurrent.CancellationException: null
> at
> java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2276)
> ~[?:1.8.0_302]
> at
> org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInternal(DefaultExecutionGraphCache.java:98)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.legacy.DefaultExecutionGraphCache.getExecutionGraphInfo(DefaultExecutionGraphCache.java:67)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.job.AbstractExecutionGraphHandler.handleRequest(AbstractExecutionGraphHandler.java:81)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.AbstractRestHandler.respondToRequest(AbstractRestHandler.java:83)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.AbstractHandler.respondAsLeader(AbstractHandler.java:195)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.lambda$channelRead0$0(LeaderRetrievalHandler.java:83)
> ~[flink-dist_2.12-1.14.2.jar:1.14.2]
> at java.util.Optional.ifPresent(Optional.java:159) [?:1.8.0_302]
> at
> org.apache.flink.util.OptionalConsumer.ifPresent(OptionalConsumer.java:45)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:80)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.LeaderRetrievalHandler.channelRead0(LeaderRetrievalHandler.java:49)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.router.RouterHandler.routed(RouterHandler.java:115)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:94)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.handler.router.RouterHandler.channelRead0(RouterHandler.java:55)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:238)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.runtime.rest.FileUploadHandler.channelRead0(FileUploadHandler.java:71)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at
> org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
> [flink-dist_2.12-1.14.2.jar:1.14.2]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302]
> """
>
> What can i do to track this down?
>
> best regards,
> Sebastian