You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Parag Somani <so...@gmail.com> on 2022/01/27 08:15:41 UTC

Duplicate job submission error

Hello All,

While deploying on our one of environment, we encountered crashloopback of
job manager pod.
Env: K8s
Flink: 1.14.2

Could you suggest, how can we troubleshoot this and possible handling of
this?


exception snipper as follows:

2022-01-27 06:58:07.326 ERROR 44 --- [lt-dispatcher-4]
c.b.a.his.service.FlinkExecutorService   : Failed to execute job

org.apache.flink.util.FlinkException: Failed to execute job 'events rates
calculation'.
        at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:2056)
~[flink-streaming-java_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.program.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:137)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.program.StreamContextEnvironment.execute(StreamContextEnvironment.java:76)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1917)
~[flink-streaming-java_2.12-1.14.2.jar:1.14.2]
        at
com.bmc.ade.his.service.FlinkExecutorService.init(FlinkExecutorService.java:37)
~[health-service-1.0.00.jar:1.0.00]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method) ~[na:na]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[na:na]
        at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:na]
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
~[na:na]
        at
org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:389)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:333)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:157)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:440)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1796)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:620)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:944)
~[spring-beans-5.3.10.jar:5.3.10]
        at
org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
~[spring-context-5.3.10.jar:5.3.10]
        at
org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583)
~[spring-context-5.3.10.jar:5.3.10]
        at
org.springframework.boot.SpringApplication.refresh(SpringApplication.java:754)
~[spring-boot-2.5.5.jar:2.5.5]
        at
org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:434)
~[spring-boot-2.5.5.jar:2.5.5]
        at
org.springframework.boot.SpringApplication.run(SpringApplication.java:338)
~[spring-boot-2.5.5.jar:2.5.5]
        at
org.springframework.boot.SpringApplication.run(SpringApplication.java:1343)
~[spring-boot-2.5.5.jar:2.5.5]
        at
org.springframework.boot.SpringApplication.run(SpringApplication.java:1332)
~[spring-boot-2.5.5.jar:2.5.5]
        at
com.bmc.ade.his.HealthAndImpactServiceApplication.main(HealthAndImpactServiceApplication.java:19)
~[health-service-1.0.00.jar:1.0.00]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method) ~[na:na]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[na:na]
        at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:na]
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
~[na:na]
        at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:253)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:216)
~[flink-clients_2.12-1.14.2.jar:1.14.2]
        at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
~[na:na]
        at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na]
        at
org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:171)
~[na:na]
        at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[na:na]
        at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$withContextClassLoader$0(ClassLoadingUtils.java:41)
~[na:na]
        at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49)
~[na:na]
        at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48)
~[na:na]
        at
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
~[na:na]
        at
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
~[na:na]
        at
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
~[na:na]
        at
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
~[na:na]
        at
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)
~[na:na]
Caused by: org.apache.flink.runtime.client.DuplicateJobSubmissionException:
Job has already been submitted.
        at
org.apache.flink.runtime.client.DuplicateJobSubmissionException.ofGloballyTerminated(DuplicateJobSubmissionException.java:33)
~[flink-runtime-1.14.2.jar:1.14.2]
        at
org.apache.flink.runtime.dispatcher.Dispatcher.submitJob(Dispatcher.java:307)
~[flink-runtime-1.14.2.jar:1.14.2]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method) ~[na:na]
        at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[na:na]
        at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:na]
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
~[na:na]
        at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRpcInvocation$1(AkkaRpcActor.java:316)
~[na:na]
        at
org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:83)
~[na:na]
        at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:314)
~[na:na]
        at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:217)
~[na:na]
        at
org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
~[na:na]
        at
org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163)
~[na:na]
        at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
~[na:na]
        at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
~[na:na]
        at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
~[scala-library-2.12.10.jar:na]
        at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
~[scala-library-2.12.10.jar:na]
        at
akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[na:na]
        at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
~[scala-library-2.12.10.jar:na]
        at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.10.jar:na]
        at
scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
~[scala-library-2.12.10.jar:na]
        at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[na:na]
        at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[na:na]
        at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
~[na:na]
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) ~[na:na]
        at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[na:na]
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[na:na]
        at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[na:na]
        at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[na:na]
        ... 5 common frames omitted

Currently I have deleted its state in ZK and restart solved the issue.

Please suggest.

Thank you in advance...!

-- 
Regards,
Parag Surajmal Somani.

Re: Duplicate job submission error

Posted by Robert Metzger <me...@gmail.com>.
Hi Parag,

it seems that you are submitting a job with the same job id multiple times.
An easy fix would be generating a new job id each time you are submitting
the job.

To debug this: check out the Flink jobmanager logs, there are log messages
for every job submission.


On Thu, Jan 27, 2022 at 9:16 AM Parag Somani <so...@gmail.com> wrote:

> Hello All,
>
> While deploying on our one of environment, we encountered crashloopback of
> job manager pod.
> Env: K8s
> Flink: 1.14.2
>
> Could you suggest, how can we troubleshoot this and possible handling of
> this?
>
>
> exception snipper as follows:
>
> 2022-01-27 06:58:07.326 ERROR 44 --- [lt-dispatcher-4]
> c.b.a.his.service.FlinkExecutorService   : Failed to execute job
>
> org.apache.flink.util.FlinkException: Failed to execute job 'events rates
> calculation'.
>         at
> org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:2056)
> ~[flink-streaming-java_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.program.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:137)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.program.StreamContextEnvironment.execute(StreamContextEnvironment.java:76)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1917)
> ~[flink-streaming-java_2.12-1.14.2.jar:1.14.2]
>         at
> com.bmc.ade.his.service.FlinkExecutorService.init(FlinkExecutorService.java:37)
> ~[health-service-1.0.00.jar:1.0.00]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method) ~[na:na]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[na:na]
>         at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[na:na]
>         at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> ~[na:na]
>         at
> org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:389)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:333)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:157)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsBeforeInitialization(AbstractAutowireCapableBeanFactory.java:440)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1796)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:620)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:944)
> ~[spring-beans-5.3.10.jar:5.3.10]
>         at
> org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
> ~[spring-context-5.3.10.jar:5.3.10]
>         at
> org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583)
> ~[spring-context-5.3.10.jar:5.3.10]
>         at
> org.springframework.boot.SpringApplication.refresh(SpringApplication.java:754)
> ~[spring-boot-2.5.5.jar:2.5.5]
>         at
> org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:434)
> ~[spring-boot-2.5.5.jar:2.5.5]
>         at
> org.springframework.boot.SpringApplication.run(SpringApplication.java:338)
> ~[spring-boot-2.5.5.jar:2.5.5]
>         at
> org.springframework.boot.SpringApplication.run(SpringApplication.java:1343)
> ~[spring-boot-2.5.5.jar:2.5.5]
>         at
> org.springframework.boot.SpringApplication.run(SpringApplication.java:1332)
> ~[spring-boot-2.5.5.jar:2.5.5]
>         at
> com.bmc.ade.his.HealthAndImpactServiceApplication.main(HealthAndImpactServiceApplication.java:19)
> ~[health-service-1.0.00.jar:1.0.00]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method) ~[na:na]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[na:na]
>         at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[na:na]
>         at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> ~[na:na]
>         at
> org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:253)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:216)
> ~[flink-clients_2.12-1.14.2.jar:1.14.2]
>         at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
> ~[na:na]
>         at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na]
>         at
> org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:171)
> ~[na:na]
>         at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68)
> ~[na:na]
>         at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$withContextClassLoader$0(ClassLoadingUtils.java:41)
> ~[na:na]
>         at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49)
> ~[na:na]
>         at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48)
> ~[na:na]
>         at
> java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
> ~[na:na]
>         at
> java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
> ~[na:na]
>         at
> java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
> ~[na:na]
>         at
> java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
> ~[na:na]
>         at
> java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)
> ~[na:na]
> Caused by:
> org.apache.flink.runtime.client.DuplicateJobSubmissionException: Job has
> already been submitted.
>         at
> org.apache.flink.runtime.client.DuplicateJobSubmissionException.ofGloballyTerminated(DuplicateJobSubmissionException.java:33)
> ~[flink-runtime-1.14.2.jar:1.14.2]
>         at
> org.apache.flink.runtime.dispatcher.Dispatcher.submitJob(Dispatcher.java:307)
> ~[flink-runtime-1.14.2.jar:1.14.2]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method) ~[na:na]
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[na:na]
>         at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[na:na]
>         at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> ~[na:na]
>         at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRpcInvocation$1(AkkaRpcActor.java:316)
> ~[na:na]
>         at
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:83)
> ~[na:na]
>         at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:314)
> ~[na:na]
>         at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:217)
> ~[na:na]
>         at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78)
> ~[na:na]
>         at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163)
> ~[na:na]
>         at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
> ~[na:na]
>         at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
> ~[na:na]
>         at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
> ~[scala-library-2.12.10.jar:na]
>         at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
> ~[scala-library-2.12.10.jar:na]
>         at
> akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[na:na]
>         at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
> ~[scala-library-2.12.10.jar:na]
>         at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> ~[scala-library-2.12.10.jar:na]
>         at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
> ~[scala-library-2.12.10.jar:na]
>         at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[na:na]
>         at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[na:na]
>         at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
> ~[na:na]
>         at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
> ~[na:na]
>         at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[na:na]
>         at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[na:na]
>         at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[na:na]
>         at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[na:na]
>         ... 5 common frames omitted
>
> Currently I have deleted its state in ZK and restart solved the issue.
>
> Please suggest.
>
> Thank you in advance...!
>
> --
> Regards,
> Parag Surajmal Somani.
>