You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flink.apache.org by Dan Hill <qu...@gmail.com> on 2022/04/10 20:31:04 UTC

Flink job savepoints but job didn't terminate successfully

This happened for 2 jobs in a row.  We just migrated to Flink v1.14.4.

I savepointed but the job didn't terminate successfully.

I looked in Jira for this error message.  I'm guessing this is a different
issue since I'm on Flink v1.14.4.

https://issues.apache.org/jira/browse/FLINK-24846?jql=text%20~%20%22%5C%22Inconsistent%20execution%20state%20after%20stopping%20with%5C%22%22

Here's a screenshot of the JM UI.  Ignore the previous failed savepoint.
The exception is down below.

[image: Screen Shot 2022-04-10 at 12.59.16 PM.png]

Exception
2022-04-10 20:06:00,116 WARN org.apache.flink.runtime.jobmaster.JobMaster
[] - A savepoint was created at
s3a://my-flink-state/savepoints/savepoint-dbbf12-b0d75948d040
but the corresponding job dbbf122f0d247a1ab496691cedfd8fff didn't terminate
successfully.
org.apache.flink.util.FlinkException: Inconsistent execution state after
stopping with savepoint. At least one execution is still in one of the
following states: FAILED, CANCELED. A global fail-over is triggered to
recover the job dbbf122f0d247a1ab496691cedfd8fff.
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl
.terminateExceptionallyWithGlobalFailover(
StopWithSavepointTerminationHandlerImpl.java:174) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.access$500(
StopWithSavepointTerminationHandlerImpl.java:54) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl$SavepointCreated
.onAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:239)
~[flink-dist_2.12-1.14.4.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.handleAnyExecutionNotFinished(
StopWithSavepointTerminationHandlerImpl.java:151) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.handleExecutionsTermination(
StopWithSavepointTerminationHandlerImpl.java:112) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:
670) ~[?:1.8.0_322]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(
CompletableFuture.java:646) ~[?:1.8.0_322]
at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture
.java:456) ~[?:1.8.0_322]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(
AkkaRpcActor.java:455) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils
.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(
AkkaRpcActor.java:455) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(
AkkaRpcActor.java:213) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(
FencedAkkaRpcActor.java:78) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor
.java:163) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.Actor.aroundReceive(Actor.scala:537) [flink-rpc-akka_8bd98a79-
85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.run(Mailbox.scala:231) [flink-rpc-akka_8bd98a79-85
ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.exec(Mailbox.scala:243) [flink-rpc-akka_8bd98a79-85
ae-4454-a22e-2385fac31150.jar:1.14.4]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.
0_322]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:
1056) [?:1.8.0_322]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:
1.8.0_322]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:
175) [?:1.8.0_322]
2022-04-10 20:06:00,117 INFO org.apache.flink.runtime.jobmaster.JobMaster
[] - Trying to recover from a global failure.
org.apache.flink.util.FlinkException: Inconsistent execution state after
stopping with savepoint. At least one execution is still in one of the
following states: FAILED, CANCELED. A global fail-over is triggered to
recover the job dbbf122f0d247a1ab496691cedfd8fff.
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl
.terminateExceptionallyWithGlobalFailover(
StopWithSavepointTerminationHandlerImpl.java:174) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.access$500(
StopWithSavepointTerminationHandlerImpl.java:54) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl$SavepointCreated
.onAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:239)
~[flink-dist_2.12-1.14.4.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.handleAnyExecutionNotFinished(
StopWithSavepointTerminationHandlerImpl.java:151) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at org.apache.flink.runtime.scheduler.stopwithsavepoint.
StopWithSavepointTerminationHandlerImpl.handleExecutionsTermination(
StopWithSavepointTerminationHandlerImpl.java:112) ~[flink-dist_2.12-1.14.4
.jar:1.14.4]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:
670) ~[?:1.8.0_322]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(
CompletableFuture.java:646) ~[?:1.8.0_322]
at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture
.java:456) ~[?:1.8.0_322]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(
AkkaRpcActor.java:455) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils
.runWithContextClassLoader(ClassLoadingUtils.java:68)
~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(
AkkaRpcActor.java:455) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(
AkkaRpcActor.java:213) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(
FencedAkkaRpcActor.java:78) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385f
ac31150.jar:1.14.4]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor
.java:163) ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:123)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.Actor.aroundReceive(Actor.scala:537) [flink-rpc-akka_8bd98a79-
85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.Actor.aroundReceive$(Actor.scala:535)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.actor.ActorCell.invoke(ActorCell.scala:548)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270)
[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.run(Mailbox.scala:231) [flink-rpc-akka_8bd98a79-85
ae-4454-a22e-2385fac31150.jar:1.14.4]
at akka.dispatch.Mailbox.exec(Mailbox.scala:243) [flink-rpc-akka_8bd98a79-85
ae-4454-a22e-2385fac31150.jar:1.14.4]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.
0_322]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:
1056) [?:1.8.0_322]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:
1.8.0_322]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:
175) [?:1.8.0_322]

Re: Flink job savepoints but job didn't terminate successfully

Posted by Chesnay Schepler <ch...@apache.org>.
We'll have to figure which execution did not shutdown and why that is 
the case.
This will require digging through the logs.

To find out what unexpected execution state we're dealing with, search 
for this log message:
"Stop-with-savepoint transitioned from {} to {} on execution termination 
handling for job {} with some executions being in an not-finished state: {}"

Then check which executions transitioned into that state, then search to 
the TM logs for any exceptions for that execution.


On 10/04/2022 22:31, Dan Hill wrote:
> This happened for 2 jobs in a row.  We just migrated to Flink v1.14.4.
>
> I savepointed but the job didn't terminate successfully.
>
> I looked in Jira for this error message.  I'm guessing this is a 
> different issue since I'm on Flink v1.14.4.
> https://issues.apache.org/jira/browse/FLINK-24846?jql=text%20~%20%22%5C%22Inconsistent%20execution%20state%20after%20stopping%20with%5C%22%22
> <https://issues.apache.org/jira/browse/FLINK-24846?jql=text%20~%20%22%5C%22Inconsistent%20execution%20state%20after%20stopping%20with%5C%22%22>
>
> Here's a screenshot of the JM UI.  Ignore the previous failed 
> savepoint.  The exception is down below.
>
> Screen Shot 2022-04-10 at 12.59.16 PM.png
>
> Exception
> 2022-04-10 20:06:00,116 WARN 
> org.apache.flink.runtime.jobmaster.JobMaster [] - A savepoint was 
> created at 
> s3a://my-flink-state/savepoints/savepoint-dbbf12-b0d75948d040 but the 
> corresponding job dbbf122f0d247a1ab496691cedfd8fff didn't terminate 
> successfully.
> org.apache.flink.util.FlinkException: Inconsistent execution state 
> after stopping with savepoint. At least one execution is still in one 
> of the following states: FAILED, CANCELED. A global fail-over is 
> triggered to recover the job dbbf122f0d247a1ab496691cedfd8fff.
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.terminateExceptionallyWithGlobalFailover(StopWithSavepointTerminationHandlerImpl.java:174) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.access$500(StopWithSavepointTerminationHandlerImpl.java:54) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl$SavepointCreated.onAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:239) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.handleAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:151) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.handleExecutionsTermination(StopWithSavepointTerminationHandlerImpl.java:112) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:670) 
> ~[?:1.8.0_322]
> at 
> java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:646) 
> ~[?:1.8.0_322]
> at 
> java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) 
> ~[?:1.8.0_322]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:455) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:455) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:213) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.apply(CaseStatements.scala:24) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.apply(CaseStatements.scala:20) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.Actor.aroundReceive(Actor.scala:537) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.Actor.aroundReceive$(Actor.scala:535) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.ActorCell.invoke(ActorCell.scala:548) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.run(Mailbox.scala:231) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.exec(Mailbox.scala:243) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) 
> [?:1.8.0_322]
> at 
> java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) 
> [?:1.8.0_322]
> at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) 
> [?:1.8.0_322]
> at 
> java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) 
> [?:1.8.0_322]
> 2022-04-10 20:06:00,117 INFO 
> org.apache.flink.runtime.jobmaster.JobMaster [] - Trying to recover 
> from a global failure.
> org.apache.flink.util.FlinkException: Inconsistent execution state 
> after stopping with savepoint. At least one execution is still in one 
> of the following states: FAILED, CANCELED. A global fail-over is 
> triggered to recover the job dbbf122f0d247a1ab496691cedfd8fff.
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.terminateExceptionallyWithGlobalFailover(StopWithSavepointTerminationHandlerImpl.java:174) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.access$500(StopWithSavepointTerminationHandlerImpl.java:54) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl$SavepointCreated.onAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:239) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.handleAnyExecutionNotFinished(StopWithSavepointTerminationHandlerImpl.java:151) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> org.apache.flink.runtime.scheduler.stopwithsavepoint.StopWithSavepointTerminationHandlerImpl.handleExecutionsTermination(StopWithSavepointTerminationHandlerImpl.java:112) 
> ~[flink-dist_2.12-1.14.4.jar:1.14.4]
> at 
> java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:670) 
> ~[?:1.8.0_322]
> at 
> java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:646) 
> ~[?:1.8.0_322]
> at 
> java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) 
> ~[?:1.8.0_322]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:455) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:455) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:213) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at 
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163) 
> ~[flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.apply(CaseStatements.scala:24) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.apply(CaseStatements.scala:20) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.japi.pf 
> <http://akka.japi.pf>.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.Actor.aroundReceive(Actor.scala:537) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.Actor.aroundReceive$(Actor.scala:535) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.actor.ActorCell.invoke(ActorCell.scala:548) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.run(Mailbox.scala:231) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at akka.dispatch.Mailbox.exec(Mailbox.scala:243) 
> [flink-rpc-akka_8bd98a79-85ae-4454-a22e-2385fac31150.jar:1.14.4]
> at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) 
> [?:1.8.0_322]
> at 
> java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) 
> [?:1.8.0_322]
> at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) 
> [?:1.8.0_322]
> at 
> java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) 
> [?:1.8.0_322]