You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@flink.apache.org by Ufuk Celebi <uc...@apache.org> on 2015/03/25 13:50:31 UTC
TaskManagerFailsITCase gets stuck
I saw a similar issue yesterday as well:
The following test gets stuck: TaskManagerFailsITCase should handle hard failing task manager
Apparently, the task managers never registers. Can someone confirm this from the stack trace? Did someone run into this as well?
$ jps 23800
surefirebooter6764955154990604015.jar
$ jstack 23800
2015-03-25 13:46:44
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.20-b23 mixed mode):
"Flink Netty Server (1524) Thread 0" #97 daemon prio=5 os_prio=31 tid=0x00007f99269c2000 nid=0x790b runnable [0x000000019010c000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000134284700> (a io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000134284638> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000134284518> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)
"Flink Netty Server (1525) Thread 0" #96 daemon prio=5 os_prio=31 tid=0x00007f99229a3800 nid=0x8f0b runnable [0x0000000190009000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000001350f8438> (a io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x00000001350f8370> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000001350f8250> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)
"flink-akka.actor.default-dispatcher-5" #88 daemon prio=5 os_prio=31 tid=0x00007f99229cf800 nid=0x3b07 waiting on condition [0x000000018f056000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000011e0701d0> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"Timer-1" #79 daemon prio=5 os_prio=31 tid=0x00007f99231a3000 nid=0x610b in Object.wait() [0x000000018e437000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000011e073058> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x000000011e073058> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"BLOB Server listener at 59112" #77 daemon prio=5 os_prio=31 tid=0x00007f9926927000 nid=0x9a07 runnable [0x000000018e334000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
at java.net.ServerSocket.implAccept(ServerSocket.java:545)
at java.net.ServerSocket.accept(ServerSocket.java:513)
at org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
"flink-scheduler-1" #73 daemon prio=5 os_prio=31 tid=0x00007f9924074000 nid=0x8a0b waiting on condition [0x000000018ce94000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
at akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
at java.lang.Thread.run(Thread.java:745)
"Attach Listener" #70 daemon prio=9 os_prio=31 tid=0x00007f992591a000 nid=0x820b waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Flink Netty Server (1524) Thread 0" #65 daemon prio=5 os_prio=31 tid=0x00007f992619f800 nid=0xa603 runnable [0x000000018f25c000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000113237b98> (a io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000113237bb8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000113237b48> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)
"Flink Netty Server (1525) Thread 0" #67 daemon prio=5 os_prio=31 tid=0x00007f992619e800 nid=0xa407 runnable [0x000000018caa4000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000011321d258> (a io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x000000011321d278> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000011321d208> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)
"ForkJoinPool-1-worker-1" #59 daemon prio=5 os_prio=31 tid=0x00007f9926930000 nid=0x9c03 waiting on condition [0x000000018ee50000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000112e486f8> (a scala.concurrent.forkjoin.ForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"flink-akka.actor.default-dispatcher-5" #55 daemon prio=5 os_prio=31 tid=0x00007f9922a14800 nid=0x9403 waiting on condition [0x000000018ea44000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000112f9d870> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"Timer-0" #45 daemon prio=5 os_prio=31 tid=0x00007f9923a45800 nid=0x8407 in Object.wait() [0x000000018e02c000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000112f945f8> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000112f945f8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"BLOB Server listener at 58482" #43 daemon prio=5 os_prio=31 tid=0x00007f9925933800 nid=0x7a07 runnable [0x000000018df29000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
at java.net.ServerSocket.implAccept(ServerSocket.java:545)
at java.net.ServerSocket.accept(ServerSocket.java:513)
at org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
"flink-scheduler-1" #40 daemon prio=5 os_prio=31 tid=0x00007f99251a6800 nid=0x7e07 waiting on condition [0x000000018d063000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
at akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
at java.lang.Thread.run(Thread.java:745)
"New I/O server boss #6" #23 daemon prio=5 os_prio=31 tid=0x00007f99260e6000 nid=0x7103 runnable [0x000000018da20000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c0f118> (a sun.nio.ch.Util$2)
- locked <0x0000000112c0f128> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c0f0c8> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"New I/O worker #5" #22 daemon prio=5 os_prio=31 tid=0x00007f9925093800 nid=0x6f03 runnable [0x000000018d91d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c131c0> (a sun.nio.ch.Util$2)
- locked <0x0000000112c131d0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c13170> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"New I/O worker #4" #21 daemon prio=5 os_prio=31 tid=0x00007f992610b800 nid=0x6d03 runnable [0x000000018d81a000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c17758> (a sun.nio.ch.Util$2)
- locked <0x0000000112c17768> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c17708> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"New I/O boss #3" #20 daemon prio=5 os_prio=31 tid=0x00007f99268cc800 nid=0x6b03 runnable [0x000000018d717000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c1f438> (a sun.nio.ch.Util$2)
- locked <0x0000000112c1f448> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c1f3e8> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"New I/O worker #2" #18 daemon prio=5 os_prio=31 tid=0x00007f99258b7000 nid=0x6903 runnable [0x000000018d614000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c270e0> (a sun.nio.ch.Util$2)
- locked <0x0000000112c270f0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c27090> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"New I/O worker #1" #17 daemon prio=5 os_prio=31 tid=0x00007f99268e7800 nid=0x6703 runnable [0x000000018d511000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000112c2ea70> (a sun.nio.ch.Util$2)
- locked <0x0000000112c2ea80> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000112c2ea20> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"TestingActorSystem-akka.remote.default-remote-dispatcher-6" #15 daemon prio=5 os_prio=31 tid=0x00007f992603d800 nid=0x6503 waiting on condition [0x000000018d20e000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000112c35b70> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"TestingActorSystem-akka.remote.default-remote-dispatcher-5" #14 daemon prio=5 os_prio=31 tid=0x00007f99258b6800 nid=0x6303 waiting on condition [0x000000018c1a5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000112c35b70> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"TestingActorSystem-akka.actor.default-dispatcher-2" #11 daemon prio=5 os_prio=31 tid=0x00007f9923bba800 nid=0x5d03 waiting on condition [0x000000018cc8e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000112c17998> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"TestingActorSystem-scheduler-1" #10 daemon prio=5 os_prio=31 tid=0x00007f9924169000 nid=0x5b03 waiting on condition [0x000000018c709000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
at akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f9926802000 nid=0x5703 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f9923807800 nid=0x5503 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f992400d800 nid=0x5303 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f9923807000 nid=0x5103 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f992281e000 nid=0x4f03 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f992281c800 nid=0x4d03 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f9923806000 nid=0x3703 in Object.wait() [0x000000018ab5c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000112c02ad0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142)
- locked <0x0000000112c02ad0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f9922813800 nid=0x3503 in Object.wait() [0x000000018aa59000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
- locked <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
"main-ScalaTest-running-TaskManagerFailsITCase" #1 prio=5 os_prio=31 tid=0x00007f9924001800 nid=0x1303 waiting on condition [0x0000000103592000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000011e07db10> (a scala.concurrent.impl.Promise$CompletionLatch)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
at scala.concurrent.Await$.ready(package.scala:86)
at org.apache.flink.runtime.minicluster.FlinkMiniCluster.waitForTaskManagersToBeRegistered(FlinkMiniCluster.scala:179)
at org.apache.flink.runtime.minicluster.FlinkMiniCluster.<init>(FlinkMiniCluster.scala:78)
at org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster.<init>(LocalFlinkMiniCluster.scala:42)
at org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:41)
at org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:44)
at org.apache.flink.test.util.ForkableFlinkMiniCluster$.startCluster(ForkableFlinkMiniCluster.scala:150)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply$mcV$sp(TaskManagerFailsITCase.scala:144)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.WordSpecLike$$anon$1.apply(WordSpecLike.scala:953)
at org.scalatest.Suite$class.withFixture(Suite.scala:1122)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.withFixture(TaskManagerFailsITCase.scala:43)
at org.scalatest.WordSpecLike$class.invokeWithFixture$1(WordSpecLike.scala:950)
at org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
at org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
at org.scalatest.WordSpecLike$class.runTest(WordSpecLike.scala:962)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTest(TaskManagerFailsITCase.scala:43)
at org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
at org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
at scala.collection.immutable.List.foreach(List.scala:318)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427)
at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
at scala.collection.immutable.List.foreach(List.scala:318)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)
at org.scalatest.WordSpecLike$class.runTests(WordSpecLike.scala:1021)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTests(TaskManagerFailsITCase.scala:43)
at org.scalatest.Suite$class.run(Suite.scala:1424)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org$scalatest$WordSpecLike$$super$run(TaskManagerFailsITCase.scala:43)
at org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
at org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
at org.scalatest.SuperEngine.runImpl(Engine.scala:545)
at org.scalatest.WordSpecLike$class.run(WordSpecLike.scala:1067)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org$scalatest$BeforeAndAfterAll$$super$run(TaskManagerFailsITCase.scala:43)
at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257)
at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256)
at org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.run(TaskManagerFailsITCase.scala:43)
at org.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
"VM Thread" os_prio=31 tid=0x00007f992280e800 nid=0x3303 runnable
"GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f9923002800 nid=0x1c03 runnable
"GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f9923003000 nid=0x1e03 runnable
"GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f9923003800 nid=0x2003 runnable
"GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f9923004800 nid=0x2203 runnable
"GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f9925000000 nid=0x2403 runnable
"GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f992280b800 nid=0x2603 runnable
"GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f9923005000 nid=0x2803 runnable
"GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f9923005800 nid=0x2d03 runnable
"VM Periodic Task Thread" os_prio=31 tid=0x00007f9923000800 nid=0x5903 waiting on condition
JNI global references: 127
Re: TaskManagerFailsITCase gets stuck
Posted by Henry Saputra <he...@gmail.com>.
Just pull from master and saw different error when running mvn test.
I just posted it to dev@ list.
On Wed, Mar 25, 2015 at 7:05 AM, Till Rohrmann <ti...@gmail.com> wrote:
> What do the logs say? It looks as if there is some issue with the
> TaskManager start up because the main is in the method
> waitForTaskManagersToBeRegistered. This happens for the initial
> ForkableFlinkMiniCluster start.
>
> On Wed, Mar 25, 2015 at 1:50 PM, Ufuk Celebi <uc...@apache.org> wrote:
>
>> I saw a similar issue yesterday as well:
>>
>> The following test gets stuck: TaskManagerFailsITCase should handle hard
>> failing task manager
>>
>> Apparently, the task managers never registers. Can someone confirm this
>> from the stack trace? Did someone run into this as well?
>>
>> $ jps 23800
>> surefirebooter6764955154990604015.jar
>> $ jstack 23800
>>
>> 2015-03-25 13:46:44
>> Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.20-b23 mixed mode):
>>
>> "Flink Netty Server (1524) Thread 0" #97 daemon prio=5 os_prio=31
>> tid=0x00007f99269c2000 nid=0x790b runnable [0x000000019010c000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000134284700> (a
>> io.netty.channel.nio.SelectedSelectionKeySet)
>> - locked <0x0000000134284638> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000134284518> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
>> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
>> at
>> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "Flink Netty Server (1525) Thread 0" #96 daemon prio=5 os_prio=31
>> tid=0x00007f99229a3800 nid=0x8f0b runnable [0x0000000190009000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x00000001350f8438> (a
>> io.netty.channel.nio.SelectedSelectionKeySet)
>> - locked <0x00000001350f8370> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x00000001350f8250> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
>> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
>> at
>> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "flink-akka.actor.default-dispatcher-5" #88 daemon prio=5 os_prio=31
>> tid=0x00007f99229cf800 nid=0x3b07 waiting on condition [0x000000018f056000]
>> java.lang.Thread.State: WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x000000011e0701d0> (a
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "Timer-1" #79 daemon prio=5 os_prio=31 tid=0x00007f99231a3000 nid=0x610b
>> in Object.wait() [0x000000018e437000]
>> java.lang.Thread.State: TIMED_WAITING (on object monitor)
>> at java.lang.Object.wait(Native Method)
>> - waiting on <0x000000011e073058> (a java.util.TaskQueue)
>> at java.util.TimerThread.mainLoop(Timer.java:552)
>> - locked <0x000000011e073058> (a java.util.TaskQueue)
>> at java.util.TimerThread.run(Timer.java:505)
>>
>> "BLOB Server listener at 59112" #77 daemon prio=5 os_prio=31
>> tid=0x00007f9926927000 nid=0x9a07 runnable [0x000000018e334000]
>> java.lang.Thread.State: RUNNABLE
>> at java.net.PlainSocketImpl.socketAccept(Native Method)
>> at
>> java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
>> at java.net.ServerSocket.implAccept(ServerSocket.java:545)
>> at java.net.ServerSocket.accept(ServerSocket.java:513)
>> at
>> org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
>>
>> "flink-scheduler-1" #73 daemon prio=5 os_prio=31 tid=0x00007f9924074000
>> nid=0x8a0b waiting on condition [0x000000018ce94000]
>> java.lang.Thread.State: TIMED_WAITING (sleeping)
>> at java.lang.Thread.sleep(Native Method)
>> at
>> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "Attach Listener" #70 daemon prio=9 os_prio=31 tid=0x00007f992591a000
>> nid=0x820b waiting on condition [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "Flink Netty Server (1524) Thread 0" #65 daemon prio=5 os_prio=31
>> tid=0x00007f992619f800 nid=0xa603 runnable [0x000000018f25c000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000113237b98> (a
>> io.netty.channel.nio.SelectedSelectionKeySet)
>> - locked <0x0000000113237bb8> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000113237b48> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
>> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
>> at
>> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "Flink Netty Server (1525) Thread 0" #67 daemon prio=5 os_prio=31
>> tid=0x00007f992619e800 nid=0xa407 runnable [0x000000018caa4000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x000000011321d258> (a
>> io.netty.channel.nio.SelectedSelectionKeySet)
>> - locked <0x000000011321d278> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x000000011321d208> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
>> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
>> at
>> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "ForkJoinPool-1-worker-1" #59 daemon prio=5 os_prio=31
>> tid=0x00007f9926930000 nid=0x9c03 waiting on condition [0x000000018ee50000]
>> java.lang.Thread.State: WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x0000000112e486f8> (a
>> scala.concurrent.forkjoin.ForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "flink-akka.actor.default-dispatcher-5" #55 daemon prio=5 os_prio=31
>> tid=0x00007f9922a14800 nid=0x9403 waiting on condition [0x000000018ea44000]
>> java.lang.Thread.State: WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x0000000112f9d870> (a
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "Timer-0" #45 daemon prio=5 os_prio=31 tid=0x00007f9923a45800 nid=0x8407
>> in Object.wait() [0x000000018e02c000]
>> java.lang.Thread.State: TIMED_WAITING (on object monitor)
>> at java.lang.Object.wait(Native Method)
>> - waiting on <0x0000000112f945f8> (a java.util.TaskQueue)
>> at java.util.TimerThread.mainLoop(Timer.java:552)
>> - locked <0x0000000112f945f8> (a java.util.TaskQueue)
>> at java.util.TimerThread.run(Timer.java:505)
>>
>> "BLOB Server listener at 58482" #43 daemon prio=5 os_prio=31
>> tid=0x00007f9925933800 nid=0x7a07 runnable [0x000000018df29000]
>> java.lang.Thread.State: RUNNABLE
>> at java.net.PlainSocketImpl.socketAccept(Native Method)
>> at
>> java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
>> at java.net.ServerSocket.implAccept(ServerSocket.java:545)
>> at java.net.ServerSocket.accept(ServerSocket.java:513)
>> at
>> org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
>>
>> "flink-scheduler-1" #40 daemon prio=5 os_prio=31 tid=0x00007f99251a6800
>> nid=0x7e07 waiting on condition [0x000000018d063000]
>> java.lang.Thread.State: TIMED_WAITING (sleeping)
>> at java.lang.Thread.sleep(Native Method)
>> at
>> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O server boss #6" #23 daemon prio=5 os_prio=31
>> tid=0x00007f99260e6000 nid=0x7103 runnable [0x000000018da20000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c0f118> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c0f128> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c0f0c8> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
>> at
>> org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O worker #5" #22 daemon prio=5 os_prio=31 tid=0x00007f9925093800
>> nid=0x6f03 runnable [0x000000018d91d000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c131c0> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c131d0> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c13170> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at
>> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O worker #4" #21 daemon prio=5 os_prio=31 tid=0x00007f992610b800
>> nid=0x6d03 runnable [0x000000018d81a000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c17758> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c17768> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c17708> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at
>> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O boss #3" #20 daemon prio=5 os_prio=31 tid=0x00007f99268cc800
>> nid=0x6b03 runnable [0x000000018d717000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c1f438> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c1f448> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c1f3e8> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at
>> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O worker #2" #18 daemon prio=5 os_prio=31 tid=0x00007f99258b7000
>> nid=0x6903 runnable [0x000000018d614000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c270e0> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c270f0> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c27090> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at
>> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "New I/O worker #1" #17 daemon prio=5 os_prio=31 tid=0x00007f99268e7800
>> nid=0x6703 runnable [0x000000018d511000]
>> java.lang.Thread.State: RUNNABLE
>> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
>> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
>> at
>> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
>> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>> - locked <0x0000000112c2ea70> (a sun.nio.ch.Util$2)
>> - locked <0x0000000112c2ea80> (a
>> java.util.Collections$UnmodifiableSet)
>> - locked <0x0000000112c2ea20> (a sun.nio.ch.KQueueSelectorImpl)
>> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>> at
>> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
>> at
>> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
>> at
>> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
>> at
>> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
>> at
>> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "TestingActorSystem-akka.remote.default-remote-dispatcher-6" #15 daemon
>> prio=5 os_prio=31 tid=0x00007f992603d800 nid=0x6503 waiting on condition
>> [0x000000018d20e000]
>> java.lang.Thread.State: TIMED_WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x0000000112c35b70> (a
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "TestingActorSystem-akka.remote.default-remote-dispatcher-5" #14 daemon
>> prio=5 os_prio=31 tid=0x00007f99258b6800 nid=0x6303 waiting on condition
>> [0x000000018c1a5000]
>> java.lang.Thread.State: WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x0000000112c35b70> (a
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "TestingActorSystem-akka.actor.default-dispatcher-2" #11 daemon prio=5
>> os_prio=31 tid=0x00007f9923bba800 nid=0x5d03 waiting on condition
>> [0x000000018cc8e000]
>> java.lang.Thread.State: WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x0000000112c17998> (a
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
>> at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>> at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>>
>> "TestingActorSystem-scheduler-1" #10 daemon prio=5 os_prio=31
>> tid=0x00007f9924169000 nid=0x5b03 waiting on condition [0x000000018c709000]
>> java.lang.Thread.State: TIMED_WAITING (sleeping)
>> at java.lang.Thread.sleep(Native Method)
>> at
>> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
>> at
>> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
>> at java.lang.Thread.run(Thread.java:745)
>>
>> "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f9926802000
>> nid=0x5703 runnable [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f9923807800
>> nid=0x5503 waiting on condition [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f992400d800
>> nid=0x5303 waiting on condition [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f9923807000
>> nid=0x5103 waiting on condition [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f992281e000
>> nid=0x4f03 waiting on condition [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f992281c800
>> nid=0x4d03 runnable [0x0000000000000000]
>> java.lang.Thread.State: RUNNABLE
>>
>> "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f9923806000 nid=0x3703
>> in Object.wait() [0x000000018ab5c000]
>> java.lang.Thread.State: WAITING (on object monitor)
>> at java.lang.Object.wait(Native Method)
>> - waiting on <0x0000000112c02ad0> (a
>> java.lang.ref.ReferenceQueue$Lock)
>> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142)
>> - locked <0x0000000112c02ad0> (a java.lang.ref.ReferenceQueue$Lock)
>> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158)
>> at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
>>
>> "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f9922813800
>> nid=0x3503 in Object.wait() [0x000000018aa59000]
>> java.lang.Thread.State: WAITING (on object monitor)
>> at java.lang.Object.wait(Native Method)
>> - waiting on <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
>> at java.lang.Object.wait(Object.java:502)
>> at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
>> - locked <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
>>
>> "main-ScalaTest-running-TaskManagerFailsITCase" #1 prio=5 os_prio=31
>> tid=0x00007f9924001800 nid=0x1303 waiting on condition [0x0000000103592000]
>> java.lang.Thread.State: TIMED_WAITING (parking)
>> at sun.misc.Unsafe.park(Native Method)
>> - parking to wait for <0x000000011e07db10> (a
>> scala.concurrent.impl.Promise$CompletionLatch)
>> at
>> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>> at
>> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
>> at
>> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
>> at
>> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
>> at
>> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
>> at
>> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
>> at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
>> at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
>> at
>> scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
>> at scala.concurrent.Await$.ready(package.scala:86)
>> at
>> org.apache.flink.runtime.minicluster.FlinkMiniCluster.waitForTaskManagersToBeRegistered(FlinkMiniCluster.scala:179)
>> at
>> org.apache.flink.runtime.minicluster.FlinkMiniCluster.<init>(FlinkMiniCluster.scala:78)
>> at
>> org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster.<init>(LocalFlinkMiniCluster.scala:42)
>> at
>> org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:41)
>> at
>> org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:44)
>> at
>> org.apache.flink.test.util.ForkableFlinkMiniCluster$.startCluster(ForkableFlinkMiniCluster.scala:150)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply$mcV$sp(TaskManagerFailsITCase.scala:144)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
>> at
>> org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
>> at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
>> at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
>> at org.scalatest.Transformer.apply(Transformer.scala:22)
>> at org.scalatest.Transformer.apply(Transformer.scala:20)
>> at org.scalatest.WordSpecLike$$anon$1.apply(WordSpecLike.scala:953)
>> at org.scalatest.Suite$class.withFixture(Suite.scala:1122)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.withFixture(TaskManagerFailsITCase.scala:43)
>> at
>> org.scalatest.WordSpecLike$class.invokeWithFixture$1(WordSpecLike.scala:950)
>> at
>> org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
>> at
>> org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
>> at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
>> at org.scalatest.WordSpecLike$class.runTest(WordSpecLike.scala:962)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTest(TaskManagerFailsITCase.scala:43)
>> at
>> org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
>> at
>> org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
>> at
>> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
>> at
>> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
>> at scala.collection.immutable.List.foreach(List.scala:318)
>> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
>> at org.scalatest.SuperEngine.org
>> $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390)
>> at
>> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427)
>> at
>> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
>> at scala.collection.immutable.List.foreach(List.scala:318)
>> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
>> at org.scalatest.SuperEngine.org
>> $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
>> at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)
>> at
>> org.scalatest.WordSpecLike$class.runTests(WordSpecLike.scala:1021)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTests(TaskManagerFailsITCase.scala:43)
>> at org.scalatest.Suite$class.run(Suite.scala:1424)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org
>> $scalatest$WordSpecLike$$super$run(TaskManagerFailsITCase.scala:43)
>> at
>> org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
>> at
>> org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
>> at org.scalatest.SuperEngine.runImpl(Engine.scala:545)
>> at org.scalatest.WordSpecLike$class.run(WordSpecLike.scala:1067)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org
>> $scalatest$BeforeAndAfterAll$$super$run(TaskManagerFailsITCase.scala:43)
>> at
>> org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257)
>> at
>> org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256)
>> at
>> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.run(TaskManagerFailsITCase.scala:43)
>> at org.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99)
>> at
>> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
>> at
>> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
>> at
>> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
>> at
>> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
>> at
>> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
>> at
>> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
>>
>> "VM Thread" os_prio=31 tid=0x00007f992280e800 nid=0x3303 runnable
>>
>> "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f9923002800
>> nid=0x1c03 runnable
>>
>> "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f9923003000
>> nid=0x1e03 runnable
>>
>> "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f9923003800
>> nid=0x2003 runnable
>>
>> "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f9923004800
>> nid=0x2203 runnable
>>
>> "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f9925000000
>> nid=0x2403 runnable
>>
>> "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f992280b800
>> nid=0x2603 runnable
>>
>> "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f9923005000
>> nid=0x2803 runnable
>>
>> "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f9923005800
>> nid=0x2d03 runnable
>>
>> "VM Periodic Task Thread" os_prio=31 tid=0x00007f9923000800 nid=0x5903
>> waiting on condition
>>
>> JNI global references: 127
>>
>>
Re: TaskManagerFailsITCase gets stuck
Posted by Till Rohrmann <ti...@gmail.com>.
What do the logs say? It looks as if there is some issue with the
TaskManager start up because the main is in the method
waitForTaskManagersToBeRegistered. This happens for the initial
ForkableFlinkMiniCluster start.
On Wed, Mar 25, 2015 at 1:50 PM, Ufuk Celebi <uc...@apache.org> wrote:
> I saw a similar issue yesterday as well:
>
> The following test gets stuck: TaskManagerFailsITCase should handle hard
> failing task manager
>
> Apparently, the task managers never registers. Can someone confirm this
> from the stack trace? Did someone run into this as well?
>
> $ jps 23800
> surefirebooter6764955154990604015.jar
> $ jstack 23800
>
> 2015-03-25 13:46:44
> Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.20-b23 mixed mode):
>
> "Flink Netty Server (1524) Thread 0" #97 daemon prio=5 os_prio=31
> tid=0x00007f99269c2000 nid=0x790b runnable [0x000000019010c000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000134284700> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000134284638> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000134284518> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "Flink Netty Server (1525) Thread 0" #96 daemon prio=5 os_prio=31
> tid=0x00007f99229a3800 nid=0x8f0b runnable [0x0000000190009000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x00000001350f8438> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x00000001350f8370> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x00000001350f8250> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "flink-akka.actor.default-dispatcher-5" #88 daemon prio=5 os_prio=31
> tid=0x00007f99229cf800 nid=0x3b07 waiting on condition [0x000000018f056000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x000000011e0701d0> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "Timer-1" #79 daemon prio=5 os_prio=31 tid=0x00007f99231a3000 nid=0x610b
> in Object.wait() [0x000000018e437000]
> java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x000000011e073058> (a java.util.TaskQueue)
> at java.util.TimerThread.mainLoop(Timer.java:552)
> - locked <0x000000011e073058> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "BLOB Server listener at 59112" #77 daemon prio=5 os_prio=31
> tid=0x00007f9926927000 nid=0x9a07 runnable [0x000000018e334000]
> java.lang.Thread.State: RUNNABLE
> at java.net.PlainSocketImpl.socketAccept(Native Method)
> at
> java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
> at java.net.ServerSocket.implAccept(ServerSocket.java:545)
> at java.net.ServerSocket.accept(ServerSocket.java:513)
> at
> org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
>
> "flink-scheduler-1" #73 daemon prio=5 os_prio=31 tid=0x00007f9924074000
> nid=0x8a0b waiting on condition [0x000000018ce94000]
> java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(Native Method)
> at
> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
> at java.lang.Thread.run(Thread.java:745)
>
> "Attach Listener" #70 daemon prio=9 os_prio=31 tid=0x00007f992591a000
> nid=0x820b waiting on condition [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "Flink Netty Server (1524) Thread 0" #65 daemon prio=5 os_prio=31
> tid=0x00007f992619f800 nid=0xa603 runnable [0x000000018f25c000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000113237b98> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000113237bb8> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000113237b48> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "Flink Netty Server (1525) Thread 0" #67 daemon prio=5 os_prio=31
> tid=0x00007f992619e800 nid=0xa407 runnable [0x000000018caa4000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000011321d258> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x000000011321d278> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x000000011321d208> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "ForkJoinPool-1-worker-1" #59 daemon prio=5 os_prio=31
> tid=0x00007f9926930000 nid=0x9c03 waiting on condition [0x000000018ee50000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000112e486f8> (a
> scala.concurrent.forkjoin.ForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "flink-akka.actor.default-dispatcher-5" #55 daemon prio=5 os_prio=31
> tid=0x00007f9922a14800 nid=0x9403 waiting on condition [0x000000018ea44000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000112f9d870> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "Timer-0" #45 daemon prio=5 os_prio=31 tid=0x00007f9923a45800 nid=0x8407
> in Object.wait() [0x000000018e02c000]
> java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000112f945f8> (a java.util.TaskQueue)
> at java.util.TimerThread.mainLoop(Timer.java:552)
> - locked <0x0000000112f945f8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "BLOB Server listener at 58482" #43 daemon prio=5 os_prio=31
> tid=0x00007f9925933800 nid=0x7a07 runnable [0x000000018df29000]
> java.lang.Thread.State: RUNNABLE
> at java.net.PlainSocketImpl.socketAccept(Native Method)
> at
> java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
> at java.net.ServerSocket.implAccept(ServerSocket.java:545)
> at java.net.ServerSocket.accept(ServerSocket.java:513)
> at
> org.apache.flink.runtime.blob.BlobServer.run(BlobServer.java:177)
>
> "flink-scheduler-1" #40 daemon prio=5 os_prio=31 tid=0x00007f99251a6800
> nid=0x7e07 waiting on condition [0x000000018d063000]
> java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(Native Method)
> at
> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O server boss #6" #23 daemon prio=5 os_prio=31
> tid=0x00007f99260e6000 nid=0x7103 runnable [0x000000018da20000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c0f118> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c0f128> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c0f0c8> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
> at
> org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #5" #22 daemon prio=5 os_prio=31 tid=0x00007f9925093800
> nid=0x6f03 runnable [0x000000018d91d000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c131c0> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c131d0> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c13170> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #4" #21 daemon prio=5 os_prio=31 tid=0x00007f992610b800
> nid=0x6d03 runnable [0x000000018d81a000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c17758> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c17768> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c17708> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O boss #3" #20 daemon prio=5 os_prio=31 tid=0x00007f99268cc800
> nid=0x6b03 runnable [0x000000018d717000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c1f438> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c1f448> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c1f3e8> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #2" #18 daemon prio=5 os_prio=31 tid=0x00007f99258b7000
> nid=0x6903 runnable [0x000000018d614000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c270e0> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c270f0> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c27090> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #1" #17 daemon prio=5 os_prio=31 tid=0x00007f99268e7800
> nid=0x6703 runnable [0x000000018d511000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
> at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
> at
> sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000112c2ea70> (a sun.nio.ch.Util$2)
> - locked <0x0000000112c2ea80> (a
> java.util.Collections$UnmodifiableSet)
> - locked <0x0000000112c2ea20> (a sun.nio.ch.KQueueSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at
> org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "TestingActorSystem-akka.remote.default-remote-dispatcher-6" #15 daemon
> prio=5 os_prio=31 tid=0x00007f992603d800 nid=0x6503 waiting on condition
> [0x000000018d20e000]
> java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000112c35b70> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "TestingActorSystem-akka.remote.default-remote-dispatcher-5" #14 daemon
> prio=5 os_prio=31 tid=0x00007f99258b6800 nid=0x6303 waiting on condition
> [0x000000018c1a5000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000112c35b70> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "TestingActorSystem-akka.actor.default-dispatcher-2" #11 daemon prio=5
> os_prio=31 tid=0x00007f9923bba800 nid=0x5d03 waiting on condition
> [0x000000018cc8e000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000112c17998> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "TestingActorSystem-scheduler-1" #10 daemon prio=5 os_prio=31
> tid=0x00007f9924169000 nid=0x5b03 waiting on condition [0x000000018c709000]
> java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(Native Method)
> at
> akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
> at java.lang.Thread.run(Thread.java:745)
>
> "Service Thread" #9 daemon prio=9 os_prio=31 tid=0x00007f9926802000
> nid=0x5703 runnable [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "C1 CompilerThread3" #8 daemon prio=9 os_prio=31 tid=0x00007f9923807800
> nid=0x5503 waiting on condition [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread2" #7 daemon prio=9 os_prio=31 tid=0x00007f992400d800
> nid=0x5303 waiting on condition [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread1" #6 daemon prio=9 os_prio=31 tid=0x00007f9923807000
> nid=0x5103 waiting on condition [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread0" #5 daemon prio=9 os_prio=31 tid=0x00007f992281e000
> nid=0x4f03 waiting on condition [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "Signal Dispatcher" #4 daemon prio=9 os_prio=31 tid=0x00007f992281c800
> nid=0x4d03 runnable [0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f9923806000 nid=0x3703
> in Object.wait() [0x000000018ab5c000]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000112c02ad0> (a
> java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:142)
> - locked <0x0000000112c02ad0> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:158)
> at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
>
> "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f9922813800
> nid=0x3503 in Object.wait() [0x000000018aa59000]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
> at java.lang.Object.wait(Object.java:502)
> at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
> - locked <0x0000000112c28138> (a java.lang.ref.Reference$Lock)
>
> "main-ScalaTest-running-TaskManagerFailsITCase" #1 prio=5 os_prio=31
> tid=0x00007f9924001800 nid=0x1303 waiting on condition [0x0000000103592000]
> java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x000000011e07db10> (a
> scala.concurrent.impl.Promise$CompletionLatch)
> at
> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> at
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
> at
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
> at
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
> at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
> at scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:86)
> at
> scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
> at scala.concurrent.Await$.ready(package.scala:86)
> at
> org.apache.flink.runtime.minicluster.FlinkMiniCluster.waitForTaskManagersToBeRegistered(FlinkMiniCluster.scala:179)
> at
> org.apache.flink.runtime.minicluster.FlinkMiniCluster.<init>(FlinkMiniCluster.scala:78)
> at
> org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster.<init>(LocalFlinkMiniCluster.scala:42)
> at
> org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:41)
> at
> org.apache.flink.test.util.ForkableFlinkMiniCluster.<init>(ForkableFlinkMiniCluster.scala:44)
> at
> org.apache.flink.test.util.ForkableFlinkMiniCluster$.startCluster(ForkableFlinkMiniCluster.scala:150)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply$mcV$sp(TaskManagerFailsITCase.scala:144)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(TaskManagerFailsITCase.scala:131)
> at
> org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
> at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
> at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> at org.scalatest.Transformer.apply(Transformer.scala:22)
> at org.scalatest.Transformer.apply(Transformer.scala:20)
> at org.scalatest.WordSpecLike$$anon$1.apply(WordSpecLike.scala:953)
> at org.scalatest.Suite$class.withFixture(Suite.scala:1122)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.withFixture(TaskManagerFailsITCase.scala:43)
> at
> org.scalatest.WordSpecLike$class.invokeWithFixture$1(WordSpecLike.scala:950)
> at
> org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
> at
> org.scalatest.WordSpecLike$$anonfun$runTest$1.apply(WordSpecLike.scala:962)
> at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
> at org.scalatest.WordSpecLike$class.runTest(WordSpecLike.scala:962)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTest(TaskManagerFailsITCase.scala:43)
> at
> org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
> at
> org.scalatest.WordSpecLike$$anonfun$runTests$1.apply(WordSpecLike.scala:1021)
> at
> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
> at
> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
> at org.scalatest.SuperEngine.org
> $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390)
> at
> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427)
> at
> org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
> at scala.collection.immutable.List.foreach(List.scala:318)
> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
> at org.scalatest.SuperEngine.org
> $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
> at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)
> at
> org.scalatest.WordSpecLike$class.runTests(WordSpecLike.scala:1021)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.runTests(TaskManagerFailsITCase.scala:43)
> at org.scalatest.Suite$class.run(Suite.scala:1424)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org
> $scalatest$WordSpecLike$$super$run(TaskManagerFailsITCase.scala:43)
> at
> org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
> at
> org.scalatest.WordSpecLike$$anonfun$run$1.apply(WordSpecLike.scala:1067)
> at org.scalatest.SuperEngine.runImpl(Engine.scala:545)
> at org.scalatest.WordSpecLike$class.run(WordSpecLike.scala:1067)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.org
> $scalatest$BeforeAndAfterAll$$super$run(TaskManagerFailsITCase.scala:43)
> at
> org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257)
> at
> org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256)
> at
> org.apache.flink.api.scala.runtime.taskmanager.TaskManagerFailsITCase.run(TaskManagerFailsITCase.scala:43)
> at org.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
> at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
>
> "VM Thread" os_prio=31 tid=0x00007f992280e800 nid=0x3303 runnable
>
> "GC task thread#0 (ParallelGC)" os_prio=31 tid=0x00007f9923002800
> nid=0x1c03 runnable
>
> "GC task thread#1 (ParallelGC)" os_prio=31 tid=0x00007f9923003000
> nid=0x1e03 runnable
>
> "GC task thread#2 (ParallelGC)" os_prio=31 tid=0x00007f9923003800
> nid=0x2003 runnable
>
> "GC task thread#3 (ParallelGC)" os_prio=31 tid=0x00007f9923004800
> nid=0x2203 runnable
>
> "GC task thread#4 (ParallelGC)" os_prio=31 tid=0x00007f9925000000
> nid=0x2403 runnable
>
> "GC task thread#5 (ParallelGC)" os_prio=31 tid=0x00007f992280b800
> nid=0x2603 runnable
>
> "GC task thread#6 (ParallelGC)" os_prio=31 tid=0x00007f9923005000
> nid=0x2803 runnable
>
> "GC task thread#7 (ParallelGC)" os_prio=31 tid=0x00007f9923005800
> nid=0x2d03 runnable
>
> "VM Periodic Task Thread" os_prio=31 tid=0x00007f9923000800 nid=0x5903
> waiting on condition
>
> JNI global references: 127
>
>