You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Martin <ma...@sonicdev.de> on 2022/01/20 12:15:46 UTC
Flink 1.14.3: Can not access job information from a jobmanager UI
Hey,
I upgraded today my Flink application to Flink 1.14.3.
I run it in a HA-Standalone-K8 deployment with 2 JobManagers, so one active
and one on standby.
As its only a prototyp I make the UI, port 8081 of the JobManager pods,
avaiable via NodePort.
Already with older Flink version I sometimes got some problems within the UI
telling "Internal server error.".
With the new 1.14.3 its really often.
It seem, that depending on to which pod the traffic is load balanced, by the
NodePort, the UI works or has the problems.
Right now, I dont know which has trouble, but due to the other issues about
that, I assume its also the stand-by one.
Here is the exception from the JobManager log, fetched from the UI, from the
time the errors happend (so from the erroneous JobManager).
2022-01-20 12:01:11,829 ERROR
org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler [] - Unhandled
exception.
org.apache.flink.runtime.rpc.akka.exceptions.AkkaRpcException: Failed to
serialize the result for RPC call : requestMultipleJobDetails.
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:417)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$sendAsyncResponse$2(AkkaRpcActor.java:373)
~[?:?]
at java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)
~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at
org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.handleCompletedFuture(FutureUtils.java:858)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.lambda$new$0(FutureUtils.java:876)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$1(AkkaInvocationHandler.java:258)
~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at
org.apache.flink.util.concurrent.FutureUtils.doForward(FutureUtils.java:1389)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$null$1(ClassLoadingUtils.java:93)
~[?:?]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[?:?]
at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$guardCompletionWithContextClassLoader$2(ClassLoadingUtils.java:92)
~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
Source) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
at
org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$1.onComplete(AkkaFutureUtils.java:47)
~[?:?]
at akka.dispatch.OnComplete.internal(Future.scala:300) ~[?:?]
at akka.dispatch.OnComplete.internal(Future.scala:297) ~[?:?]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:224) ~[?:?]
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:221) ~[?:?]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-
dist_2.12-1.14.3.jar:1.14.3]
at
org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$DirectExecutionContext.execute(AkkaFutureUtils.java:65)
~[?:?]
at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:621) ~[?:?]
at
akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:24)
~[?:?]
at
akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:23)
~[?:?]
at scala.concurrent.Future.$anonfun$andThen$1(Future.scala:532) ~[flink-
dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29) ~[flink-
dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-
dist_2.12-1.14.3.jar:1.14.3]
at
akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:63)
~[?:?]
at
akka.dispatch.BatchingExecutor$BlockableBatch.$anonfun$run$1(BatchingExecutor.scala:100)
~[?:?]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:81)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:100)
~[?:?]
at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) ~[?:?]
at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48)
~[?:?]
at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown Source)
[?:?]
at java.util.concurrent.ForkJoinPool.scan(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) [?:?]
Caused by: java.io.NotSerializableException: java.util.HashMap$Values
at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.defaultWriteFields(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeSerialData(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeOrdinaryObject(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
at java.io.ObjectOutputStream.writeObject(Unknown Source) ~[?:?]
at
org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:632)
~[flink-dist_2.12-1.14.3.jar:1.14.3]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcSerializedValue.valueOf(AkkaRpcSerializedValue.java:66)
~[?:?]
at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:400)
~[?:?]
... 54 more
Re: Flink 1.14.3: Can not access job information from a jobmanager UI
Posted by John Smith <ja...@gmail.com>.
I had the same issue in my thread it was mentioned that it was supposed to
be fixed in 1.14.3
On Thu, 20 Jan 2022 at 07:40, Martin <ma...@sonicdev.de> wrote:
> Thanks for the quick response, I assumed thats already known, but was not
> able to find the issue. Thanks :)
>
> Chesnay Schepler schrieb am 20.01.2022 13:36 (GMT +01:00):
>
> This is a bug in Flink for which I have filed a ticket:
> https://issues.apache.org/jira/browse/FLINK-25732
>
> As is you can only request the job overview from the leading jobmanager.
>
> On 20/01/2022 13:15, Martin wrote:
>
>
>
>
Re: Flink 1.14.3: Can not access job information from a jobmanager UI
Posted by Martin <ma...@sonicdev.de>.
Thanks for the quick response, I assumed thats already known, but was not able
to find the issue. Thanks :)
Chesnay Schepler schrieb am 20.01.2022 13:36 (GMT +01:00):
> This is a bug in Flink for which I have filed a ticket:
> <https://issues.apache.org/jira/browse/FLINK-25732>
>
>
>
> As is you can only request the job overview from the leading jobmanager.
>
>
>
> On 20/01/2022 13:15, Martin wrote:
>
>
Re: Flink 1.14.3: Can not access job information from a jobmanager UI
Posted by Chesnay Schepler <ch...@apache.org>.
This is a bug in Flink for which I have filed a ticket:
https://issues.apache.org/jira/browse/FLINK-25732
As is you can only request the job overview from the leading jobmanager.
On 20/01/2022 13:15, Martin wrote:
>
> Hey,
>
> I upgraded today my Flink application to Flink 1.14.3.
>
> I run it in a HA-Standalone-K8 deployment with 2 JobManagers, so one
> active and one on standby.
> As its only a prototyp I make the UI, port 8081 of the JobManager
> pods, avaiable via NodePort.
>
> Already with older Flink version I sometimes got some problems within
> the UI telling "Internal server error.".
> With the new 1.14.3 its really often.
>
> It seem, that depending on to which pod the traffic is load balanced,
> by the NodePort, the UI works or has the problems.
> Right now, I dont know which has trouble, but due to the other issues
> about that, I assume its also the stand-by one.
>
> Here is the exception from the JobManager log, fetched from the UI,
> from the time the errors happend (so from the erroneous JobManager).
>
> 2022-01-20 12:01:11,829 ERROR
> org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler [] -
> Unhandled exception.
> org.apache.flink.runtime.rpc.akka.exceptions.AkkaRpcException: Failed
> to serialize the result for RPC call : requestMultipleJobDetails.
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:417)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$sendAsyncResponse$2(AkkaRpcActor.java:373)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) ~[?:?]
> at java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown
> Source) ~[?:?]
> at java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
> at
> org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.handleCompletedFuture(FutureUtils.java:858)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.lambda$new$0(FutureUtils.java:876)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown
> Source) ~[?:?]
> at
> java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
> Source) ~[?:?]
> at java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$1(AkkaInvocationHandler.java:258)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown
> Source) ~[?:?]
> at
> java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
> Source) ~[?:?]
> at java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
> at
> org.apache.flink.util.concurrent.FutureUtils.doForward(FutureUtils.java:1389)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$null$1(ClassLoadingUtils.java:93)
> ~[?:?]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
> ~[?:?]
> at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$guardCompletionWithContextClassLoader$2(ClassLoadingUtils.java:92)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown
> Source) ~[?:?]
> at
> java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown
> Source) ~[?:?]
> at java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
> ~[?:?]
> at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]
> at
> org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$1.onComplete(AkkaFutureUtils.java:47)
> ~[?:?]
> at akka.dispatch.OnComplete.internal(Future.scala:300) ~[?:?]
> at akka.dispatch.OnComplete.internal(Future.scala:297) ~[?:?]
> at akka.dispatch.japi$CallbackBridge.apply(Future.scala:224) ~[?:?]
> at akka.dispatch.japi$CallbackBridge.apply(Future.scala:221) ~[?:?]
> at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$DirectExecutionContext.execute(AkkaFutureUtils.java:65)
> ~[?:?]
> at
> scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:621) ~[?:?]
> at
> akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:24)
> ~[?:?]
> at
> akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:23)
> ~[?:?]
> at scala.concurrent.Future.$anonfun$andThen$1(Future.scala:532)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:63)
> ~[?:?]
> at
> akka.dispatch.BatchingExecutor$BlockableBatch.$anonfun$run$1(BatchingExecutor.scala:100)
> ~[?:?]
> at
> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:81)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:100)
> ~[?:?]
> at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) ~[?:?]
> at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48)
> ~[?:?]
> at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) [?:?]
> at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown
> Source) [?:?]
> at java.util.concurrent.ForkJoinPool.scan(Unknown Source) [?:?]
> at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) [?:?]
> at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) [?:?]
> Caused by: java.io.NotSerializableException: java.util.HashMap$Values
> at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
> at java.io.ObjectOutputStream.defaultWriteFields(Unknown Source) ~[?:?]
> at java.io.ObjectOutputStream.writeSerialData(Unknown Source) ~[?:?]
> at java.io.ObjectOutputStream.writeOrdinaryObject(Unknown Source) ~[?:?]
> at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]
> at java.io.ObjectOutputStream.writeObject(Unknown Source) ~[?:?]
> at
> org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:632)
> ~[flink-dist_2.12-1.14.3.jar:1.14.3]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcSerializedValue.valueOf(AkkaRpcSerializedValue.java:66)
> ~[?:?]
> at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:400)
> ~[?:?]
> ... 54 more
>