You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by "Yun Tang (Jira)" <ji...@apache.org> on 2022/04/19 02:55:00 UTC
[jira] [Created] (FLINK-27295) UnalignedCheckpointITCase failed due to OperatorCoordinatorHolder cannot mark checkpoint
Yun Tang created FLINK-27295:
--------------------------------
Summary: UnalignedCheckpointITCase failed due to OperatorCoordinatorHolder cannot mark checkpoint
Key: FLINK-27295
URL: https://issues.apache.org/jira/browse/FLINK-27295
Project: Flink
Issue Type: Bug
Components: Runtime / Checkpointing
Affects Versions: 1.16.0
Reporter: Yun Tang
{code:java}
09:17:42,931 [flink-akka.actor.default-dispatcher-9] INFO org.apache.flink.runtime.jobmaster.JobMaster [] - Trying to recover from a global failure.java.lang.IllegalStateException: Cannot mark for checkpoint 17, already marked for checkpoint 16 at org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.Actor.aroundReceive(Actor.scala:537) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.Actor.aroundReceive$(Actor.scala:535) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.ActorCell.invoke(ActorCell.scala:548) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.run(Mailbox.scala:231) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.exec(Mailbox.scala:243) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_292] at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_292] at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_292] at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_292]09:17:42,932 [flink-akka.actor.default-dispatcher-9] INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job Flink Streaming Job (3bb746b6088ca587b3664599a7bae6e7) switched from state RUNNING to FAILING.org.apache.flink.runtime.JobException: Recovery is suppressed by FixedDelayRestartBackoffTimeStrategy(maxNumberRestartAttempts=5, backoffTimeMS=100) at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getGlobalFailureHandlingResult(ExecutionFailureHandler.java:101) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.scheduler.DefaultScheduler.handleGlobalFailure(DefaultScheduler.java:322) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:307) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164) ~[flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.Actor.aroundReceive(Actor.scala:537) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.Actor.aroundReceive$(Actor.scala:535) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.actor.ActorCell.invoke(ActorCell.scala:548) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.run(Mailbox.scala:231) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at akka.dispatch.Mailbox.exec(Mailbox.scala:243) [flink-rpc-akka_6cc3640b-2a0a-43c9-a46c-e21f0e6e55f6.jar:1.16-SNAPSHOT] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_292] at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_292] at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_292] at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_292]Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17, already marked for checkpoint 16 at org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] ... 27 more09:17:42,930 [ Checkpoint Timer] WARN org.apache.flink.runtime.checkpoint.CheckpointFailureManager [] - Failed to trigger or complete checkpoint 17 for job 3bb746b6088ca587b3664599a7bae6e7. (0 consecutive failed attempts so far)org.apache.flink.runtime.checkpoint.CheckpointException: Trigger checkpoint failure. at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$getCheckpointException$19(CheckpointCoordinator.java:2167) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_292] at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.getCheckpointException(CheckpointCoordinator.java:2166) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:938) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.onTriggerFailure(CheckpointCoordinator.java:916) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.lambda$startTriggeringCheckpoint$7(CheckpointCoordinator.java:642) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836) [?:1.8.0_292] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811) [?:1.8.0_292] at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) [?:1.8.0_292] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_292] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_292] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_292] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_292] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_292] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_292] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]Caused by: java.lang.IllegalStateException: Cannot mark for checkpoint 17, already marked for checkpoint 16 at org.apache.flink.runtime.operators.coordination.OperatorEventValve.markForCheckpoint(OperatorEventValve.java:113) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.checkpointCoordinatorInternal(OperatorCoordinatorHolder.java:302) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder.lambda$checkpointCoordinator$0(OperatorCoordinatorHolder.java:230) ~[flink-runtime-1.16-SNAPSHOT.jar:1.16-SNAPSHOT] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:444) ~[?:?] at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[?:?] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:444) ~[?:?] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:214) ~[?:?] at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) ~[?:?] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:164) ~[?:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) ~[?:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) ~[?:?] at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) ~[scala-library-2.12.7.jar:?] at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) ~[scala-library-2.12.7.jar:?] at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[?:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) ~[scala-library-2.12.7.jar:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) ~[scala-library-2.12.7.jar:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) ~[scala-library-2.12.7.jar:?] at akka.actor.Actor.aroundReceive(Actor.scala:537) ~[?:?] at akka.actor.Actor.aroundReceive$(Actor.scala:535) ~[?:?] at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) ~[?:?] at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) ~[?:?] at akka.actor.ActorCell.invoke(ActorCell.scala:548) ~[?:?] at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) ~[?:?] at akka.dispatch.Mailbox.run(Mailbox.scala:231) ~[?:?] at akka.dispatch.Mailbox.exec(Mailbox.scala:243) ~[?:?] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) ~[?:1.8.0_292] at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) ~[?:1.8.0_292] at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) ~[?:1.8.0_292] at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) ~[?:1.8.0_292] {code}
Instace: [https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=34771&view=logs&j=5c8e7682-d68f-54d1-16a2-a09310218a49&t=86f654fa-ab48-5c1a-25f4-7e7f6afb9bba]
cc [~pnowojski] , [~becket_qin]
--
This message was sent by Atlassian Jira
(v8.20.1#820001)