You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@flink.apache.org by "Jeremy DeGroot (Jira)" <ji...@apache.org> on 2022/03/09 22:24:00 UTC
[jira] [Created] (FLINK-26558) Errors when reporting on Job status
Jeremy DeGroot created FLINK-26558:
--------------------------------------
Summary: Errors when reporting on Job status
Key: FLINK-26558
URL: https://issues.apache.org/jira/browse/FLINK-26558
Project: Flink
Issue Type: Bug
Components: Runtime / REST
Affects Versions: 1.14.3
Reporter: Jeremy DeGroot
This error is showing up very frequently in my JobManager logs since I upgraded from 1.14.2 to 1.14.3. The Flink Rest dashboard also fails to load either Running or Completed jobs when this happens.
The Job Managers are HA, running on Kubernetes. The Task managers are running on Kubernetes as well, and were also upgraded to 1.14.3.
Please advise
{{2022-03-09 22:12:40,925 ERROR org.apache.flink.runtime.rest.handler.job.JobsOverviewHandler [] - Unhandled exception.}}
{{org.apache.flink.runtime.rpc.akka.exceptions.AkkaRpcException: Failed to serialize the result for RPC call : requestMultipleJobDetails.}}
{{at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:417) ~[?:?]}}
{{at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$sendAsyncResponse$2(AkkaRpcActor.java:373) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]}}
{{at org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.handleCompletedFuture(FutureUtils.java:858) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at org.apache.flink.util.concurrent.FutureUtils$ResultConjunctFuture.lambda$new$0(FutureUtils.java:876) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]}}
{{at org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$1(AkkaInvocationHandler.java:258) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]}}
{{at org.apache.flink.util.concurrent.FutureUtils.doForward(FutureUtils.java:1389) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$null$1(ClassLoadingUtils.java:93) ~[?:?]}}
{{at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[?:?]}}
{{at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$guardCompletionWithContextClassLoader$2(ClassLoadingUtils.java:92) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) ~[?:?]}}
{{at java.util.concurrent.CompletableFuture.complete(Unknown Source) ~[?:?]}}
{{at org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$1.onComplete(AkkaFutureUtils.java:47) ~[?:?]}}
{{at akka.dispatch.OnComplete.internal(Future.scala:300) ~[?:?]}}
{{at akka.dispatch.OnComplete.internal(Future.scala:297) ~[?:?]}}
{{at akka.dispatch.japi$CallbackBridge.apply(Future.scala:224) ~[?:?]}}
{{at akka.dispatch.japi$CallbackBridge.apply(Future.scala:221) ~[?:?]}}
{{at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at org.apache.flink.runtime.concurrent.akka.AkkaFutureUtils$DirectExecutionContext.execute(AkkaFutureUtils.java:65) ~[?:?]}}
{{at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:68) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1(Promise.scala:284) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.Promise$DefaultPromise.$anonfun$tryComplete$1$adapted(Promise.scala:284) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:284) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:621) ~[?:?]}}
{{at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:24) ~[?:?]}}
{{at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:23) ~[?:?]}}
{{at scala.concurrent.Future.$anonfun$andThen$1(Future.scala:532) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:63) ~[?:?]}}
{{at akka.dispatch.BatchingExecutor$BlockableBatch.$anonfun$run$1(BatchingExecutor.scala:100) ~[?:?]}}
{{at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) ~[?:?]}}
{{at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:81) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:100) ~[?:?]}}
{{at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) ~[?:?]}}
{{at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48) ~[?:?]}}
{{at java.util.concurrent.ForkJoinTask.doExec(Unknown Source) [?:?]}}
{{at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(Unknown Source) [?:?]}}
{{at java.util.concurrent.ForkJoinPool.scan(Unknown Source) [?:?]}}
{{at java.util.concurrent.ForkJoinPool.runWorker(Unknown Source) [?:?]}}
{{at java.util.concurrent.ForkJoinWorkerThread.run(Unknown Source) }}{{Caused by: java.io.NotSerializableException: java.util.HashMap$Values}}
{{at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]}}
{{at java.io.ObjectOutputStream.defaultWriteFields(Unknown Source) ~[?:?]}}
{{at java.io.ObjectOutputStream.writeSerialData(Unknown Source) ~[?:?]}}
{{at java.io.ObjectOutputStream.writeOrdinaryObject(Unknown Source) ~[?:?]}}
{{at java.io.ObjectOutputStream.writeObject0(Unknown Source) ~[?:?]}}
{{at java.io.ObjectOutputStream.writeObject(Unknown Source) ~[?:?]}}
{{at org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:632) ~[flink-dist_2.11-1.14.3.jar:1.14.3]}}
{{at org.apache.flink.runtime.rpc.akka.AkkaRpcSerializedValue.valueOf(AkkaRpcSerializedValue.java:66) ~[?:?]}}
{{at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.serializeRemoteResultAndVerifySize(AkkaRpcActor.java:400) ~[?:?]}}
{{... 54 more[?:?]}}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)