You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user-zh@flink.apache.org by 陈卓宇 <25...@qq.com.INVALID> on 2022/06/24 04:06:46 UTC
k8s ha机制不生效，jobmanager莫名的宕机
flink版本：1.13.1
ha：k8s
日志：
{"log":"2022-06-18 13:45:30,484 INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Discarding the results produced by task execution 529937f95b2f6ded2c7e78f9ac03c5b0.\n","stream":"stdout","time":"2022-06-18T05:45:30.484980772Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Discarding the results produced by task execution a4bb1dd63337d6e5135117a86451f8f1.\n","stream":"stdout","time":"2022-06-18T05:45:30.48572719Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.resourcemanager.slotmanager.DeclarativeSlotManager [] - Clearing resource requirements of job 00000000000000000000000000000000\n","stream":"stdout","time":"2022-06-18T05:45:30.485757582Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Stopping checkpoint coordinator for job 00000000000000000000000000000000.\n","stream":"stdout","time":"2022-06-18T05:45:30.485767946Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.checkpoint.DefaultCompletedCheckpointStore [] - Suspending\n","stream":"stdout","time":"2022-06-18T05:45:30.485899546Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.kubernetes.highavailability.KubernetesCheckpointIDCounter [] - Shutting down.\n","stream":"stdout","time":"2022-06-18T05:45:30.485905512Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Job 00000000000000000000000000000000 has been suspended.\n","stream":"stdout","time":"2022-06-18T05:45:30.4859199Z"}
{"log":"2022-06-18 13:45:30,485 INFO org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool [] - Releasing slot [10ef611a742f13194d3c82c2f8209239].\n","stream":"stdout","time":"2022-06-18T05:45:30.485989028Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.resourcemanager.slotmanager.DeclarativeSlotManager [] - Clearing resource requirements of job 00000000000000000000000000000000\n","stream":"stdout","time":"2022-06-18T05:45:30.486407307Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool [] - Releasing slot [7253cc6e7cbeb7f7657ede7986a2c45c].\n","stream":"stdout","time":"2022-06-18T05:45:30.486441793Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.resourcemanager.slotmanager.DeclarativeSlotManager [] - Clearing resource requirements of job 00000000000000000000000000000000\n","stream":"stdout","time":"2022-06-18T05:45:30.486564514Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool [] - Releasing slot [d4e14c2f3205c6572dbc08586a171ed3].\n","stream":"stdout","time":"2022-06-18T05:45:30.486580036Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.resourcemanager.slotmanager.DeclarativeSlotManager [] - Clearing resource requirements of job 00000000000000000000000000000000\n","stream":"stdout","time":"2022-06-18T05:45:30.486733599Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool [] - Releasing slot [3b7ed68476533c15d16815e59c83f1bc].\n","stream":"stdout","time":"2022-06-18T05:45:30.486748418Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.resourcemanager.slotmanager.DeclarativeSlotManager [] - Clearing resource requirements of job 00000000000000000000000000000000\n","stream":"stdout","time":"2022-06-18T05:45:30.486900222Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.jobmaster.JobMaster [] - Close ResourceManager connection c5a9855462876efd215e90981b5af816: Stopping JobMaster for job insert-into_default_catalog.default_database.dwd_screen_rent_house_info_1,default_catalog.default_database.dwd_screen_rent_house_info_2(00000000000000000000000000000000)..\n","stream":"stdout","time":"2022-06-18T05:45:30.486905725Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.leaderretrieval.DefaultLeaderRetrievalService [] - Stopping DefaultLeaderRetrievalService.\n","stream":"stdout","time":"2022-06-18T05:45:30.486943122Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.kubernetes.highavailability.KubernetesLeaderRetrievalDriver [] - Stopping KubernetesLeaderRetrievalDriver{configMapName='renthouse-resourcemanager-leader'}.\n","stream":"stdout","time":"2022-06-18T05:45:30.486951594Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.kubernetes.kubeclient.resources.KubernetesConfigMapWatcher [] - The watcher is closing.\n","stream":"stdout","time":"2022-06-18T05:45:30.486979366Z"}
{"log":"2022-06-18 13:45:30,486 INFO org.apache.flink.runtime.resourcemanager.active.ActiveResourceManager [] - Disconnect job manager 88b020ca5bbcf8fde047974ace174b95@akka.tcp://flink@10.42.8.61:6123/user/rpc/jobmanager_3 for job 00000000000000000000000000000000 from the resource manager.\n","stream":"stdout","time":"2022-06-18T05:45:30.4870297Z"}
{"log":"2022-06-18 13:45:30,550 INFO org.apache.flink.runtime.leaderelection.DefaultLeaderElectionService [] - Stopping DefaultLeaderElectionService.\n","stream":"stdout","time":"2022-06-18T05:45:30.550713418Z"}
{"log":"2022-06-18 13:45:30,550 INFO org.apache.flink.kubernetes.highavailability.KubernetesLeaderElectionDriver [] - Closing KubernetesLeaderElectionDriver{configMapName='renthouse-00000000000000000000000000000000-jobmanager-leader'}.\n","stream":"stdout","time":"2022-06-18T05:45:30.550783877Z"}
{"log":"2022-06-18 13:45:30,550 INFO org.apache.flink.kubernetes.kubeclient.resources.KubernetesConfigMapWatcher [] - The watcher is closing.\n","stream":"stdout","time":"2022-06-18T05:45:30.550989289Z"}
{"log":"2022-06-18 13:45:30,551 INFO org.apache.flink.runtime.jobmanager.DefaultJobGraphStore [] - Released job graph 00000000000000000000000000000000 from KubernetesStateHandleStore{configMapName='renthouse-dispatcher-leader'}.\n","stream":"stdout","time":"2022-06-18T05:45:30.55198268Z"}
{"log":"2022-06-18 13:45:30,558 WARN org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap [] - Application failed unexpectedly: \n","stream":"stdout","time":"2022-06-18T05:45:30.559908598Z"}
{"log":"java.util.concurrent.CancellationException: null\n","stream":"stdout","time":"2022-06-18T05:45:30.559943484Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2263) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.559954905Z"}
{"log":"\u0009at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.stop(ApplicationDispatcherBootstrap.java:120) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.559965298Z"}
{"log":"\u0009at org.apache.flink.runtime.dispatcher.Dispatcher.lambda$onStop$0(Dispatcher.java:273) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.559976518Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.FutureUtils.lambda$runAfterwardsAsync$18(FutureUtils.java:687) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.55998839Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.559998717Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560007595Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.56001776Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.DirectExecutorService.execute(DirectExecutorService.java:217) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.560028073Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniCompletion.claim(CompletableFuture.java:529) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560038395Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:751) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560047248Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560056052Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560080922Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560090623Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.FutureUtils$CompletionConjunctFuture.completeFuture(FutureUtils.java:991) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.560099725Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560133734Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.56014318Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560154988Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:561) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560164754Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniRun.tryFire(CompletableFuture.java:690) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560175517Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560185835Z"}
{"log":"\u0009at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560194698Z"}
{"log":"\u0009at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560203496Z"}
{"log":"\u0009at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.560212852Z"}
{"log":"2022-06-18 13:45:30,560 ERROR org.apache.flink.runtime.entrypoint.ClusterEntrypoint [] - Fatal error occurred in the cluster entrypoint.\n","stream":"stdout","time":"2022-06-18T05:45:30.560939697Z"}
{"log":"org.apache.flink.util.FlinkException: Application failed unexpectedly.\n","stream":"stdout","time":"2022-06-18T05:45:30.560978463Z"}
{"log":"\u0009at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAndShutdownClusterAsync$0(ApplicationDispatcherBootstrap.java:170) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.560990746Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:822) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561003208Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:797) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561013939Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561024285Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2265) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561034584Z"}
{"log":"\u0009at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.stop(ApplicationDispatcherBootstrap.java:120) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.561045061Z"}
{"log":"\u0009at org.apache.flink.runtime.dispatcher.Dispatcher.lambda$onStop$0(Dispatcher.java:273) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.561056685Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.FutureUtils.lambda$runAfterwardsAsync$18(FutureUtils.java:687) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.56106733Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561096821Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561123381Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561134888Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.DirectExecutorService.execute(DirectExecutorService.java:217) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.561145301Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniCompletion.claim(CompletableFuture.java:529) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.56115582Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:751) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561165825Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561175728Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.56118772Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561198899Z"}
{"log":"\u0009at org.apache.flink.runtime.concurrent.FutureUtils$CompletionConjunctFuture.completeFuture(FutureUtils.java:991) ~[flink-dist_2.11-1.13.1.jar:1.13.1]\n","stream":"stdout","time":"2022-06-18T05:45:30.561209696Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561220397Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561230388Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561240499Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:561) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561251233Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$UniRun.tryFire(CompletableFuture.java:690) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.56126338Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561273731Z"}
{"log":"\u0009at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561284435Z"}
{"log":"\u0009at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561294345Z"}
{"log":"\u0009at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561304427Z"}
{"log":"Caused by: java.util.concurrent.CancellationException\n","stream":"stdout","time":"2022-06-18T05:45:30.561314302Z"}
{"log":"\u0009at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2263) ~[?:1.8.0_202]\n","stream":"stdout","time":"2022-06-18T05:45:30.561325171Z"}
{"log":"\u0009... 22 more\n","stream":"stdout","time":"2022-06-18T05:45:30.561335492Z"}
{"log":"2022-06-18 13:45:30,562 INFO org.apache.flink.runtime.blob.BlobServer [] - Stopped BLOB server at 0.0.0.0:6124\n","stream":"stdout","time":"2022-06-18T05:45:30.56307666Z"}
{"log":"2022-06-18 13:45:30,564 INFO org.apache.flink.runtime.entrypoint.ClusterEntrypoint [] - Shutting KubernetesApplicationClusterEntrypoint down with application status UNKNOWN. Diagnostics Cluster entrypoint has been closed externally..\n","stream":"stdout","time":"2022-06-18T05:45:30.564501951Z"}