You are viewing a plain text version of this content. The canonical link for it is here.

Posted to user@spark.apache.org by "Bae, Jae Hyeon" <me...@gmail.com> on 2015/10/21 08:45:52 UTC

Spark job stuck at DataFrameReader.json() method

Hi

My Spark job had been running well for a while but since yesterday, it's
getting stuck on DataFrameReader.json() method. I am guessing that data is
a little bit changed and InferSchema is having a problem. I am attaching
thread dump of driver application, can somebody take a look at?

Thank you
Best, Jae

2015-10-21 05:54:19
Full thread dump OpenJDK 64-Bit Server VM (25.45-b02 mixed mode):

"Attach Listener" #1455 daemon prio=9 os_prio=0 tid=0x00007f822c001000
nid=0x44d waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"shuffle-server-6" #87 daemon prio=5 os_prio=0 tid=0x00007f80c800a000
nid=0x1f8b8 runnable [0x00007f811c19c000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680ac9858> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680acb8d8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680ac97c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"shuffle-server-5" #86 daemon prio=5 os_prio=0 tid=0x00007f80c8009000
nid=0x1f8b2 runnable [0x00007f825b3f4000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680acc050> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680afe028> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680acbfb8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-411" #411 daemon prio=5 os_prio=0 tid=0x00007f81ac00c000
nid=0x1f593 waiting on condition [0x00007f811c49d000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"sparkDriver-akka.actor.default-dispatcher-18" #338 daemon prio=5 os_prio=0
tid=0x00007f81e0273000 nid=0x1f592 waiting on condition [0x00007f811c59e000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"task-result-getter-3" #248 daemon prio=5 os_prio=0 tid=0x00007f813c005000
nid=0x1f591 waiting on condition [0x00007f811c69f000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808fd9b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"task-result-getter-2" #246 daemon prio=5 os_prio=0 tid=0x00007f8110005000
nid=0x1f590 waiting on condition [0x00007f811c7a0000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808fd9b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"task-result-getter-1" #244 daemon prio=5 os_prio=0 tid=0x00007f8144003000
nid=0x1f58f waiting on condition [0x00007f811c8a1000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808fd9b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"task-result-getter-0" #242 daemon prio=5 os_prio=0 tid=0x00007f8104006800
nid=0x1f58e waiting on condition [0x00007f811c9a2000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808fd9b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"shuffle-server-4" #85 daemon prio=5 os_prio=0 tid=0x00007f80c8007000
nid=0x1f58d runnable [0x00007f811caa3000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680b027b0> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680b24770> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680b02718> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"shuffle-server-3" #84 daemon prio=5 os_prio=0 tid=0x00007f80c8005800
nid=0x1f58c runnable [0x00007f811dba5000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680b28ef8> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680b2af78> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680b28e60> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"shuffle-server-2" #83 daemon prio=5 os_prio=0 tid=0x00007f80c8004000
nid=0x1f58b runnable [0x00007f811dca6000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680b2b6f0> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680b39770> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680b2b658> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"threadDeathWatcher-2-1" #240 daemon prio=1 os_prio=0
tid=0x00007f8058009000 nid=0x1f58a waiting on condition [0x00007f811eda8000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:137)
at
io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
at java.lang.Thread.run(Thread.java:745)

"shuffle-server-1" #82 daemon prio=5 os_prio=0 tid=0x00007f80c8003800
nid=0x1f589 runnable [0x00007f811eea9000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680b3def8> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680b4f1a8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680b3de60> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"sparkDriver-akka.actor.default-dispatcher-17" #148 daemon prio=5 os_prio=0
tid=0x00007f81e0174800 nid=0x1f516 waiting on condition [0x00007f811f1aa000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-16" #147 daemon prio=5 os_prio=0
tid=0x00007f81dc001000 nid=0x1f515 waiting on condition [0x00007f811f2ab000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at
scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-15" #146 daemon prio=5 os_prio=0
tid=0x00007f8074001000 nid=0x1f514 waiting on condition [0x00007f811f3ac000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-14" #145 daemon prio=5 os_prio=0
tid=0x00007f81e8005000 nid=0x1f513 waiting on condition [0x00007f81341ee000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"BoneCP-pool-watch-thread" #118 daemon prio=5 os_prio=0
tid=0x00007f82b5264800 nid=0x1f501 waiting on condition [0x00007f8134aef000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681d60ca8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-keep-alive-scheduler" #117 daemon prio=5 os_prio=0
tid=0x00007f82b4723800 nid=0x1f500 waiting on condition [0x00007f8134bf0000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681d70538> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"com.google.common.base.internal.Finalizer" #116 daemon prio=5 os_prio=0
tid=0x00007f82b5268800 nid=0x1f4ff in Object.wait() [0x00007f8134cf1000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x0000000681d964d8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-pool-watch-thread" #115 daemon prio=5 os_prio=0
tid=0x00007f82b4ddc800 nid=0x1f4fe waiting on condition [0x00007f81351f2000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681d70798> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-keep-alive-scheduler" #114 daemon prio=5 os_prio=0
tid=0x00007f82b5924800 nid=0x1f4fd waiting on condition [0x00007f81352f3000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681da64b0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"com.google.common.base.internal.Finalizer" #113 daemon prio=5 os_prio=0
tid=0x00007f82b5923000 nid=0x1f4fc in Object.wait() [0x00007f81353f4000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x0000000681d60f38> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
at java.lang.Thread.run(Thread.java:745)

"derby.rawStoreDaemon" #112 daemon prio=5 os_prio=0 tid=0x00007f82b530d800
nid=0x1f4fb in Object.wait() [0x00007f81356f5000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
- locked <0x000000068143fc30> (a
org.apache.derby.impl.services.daemon.BasicDaemon)
at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
at java.lang.Thread.run(Thread.java:745)

"Timer-2" #111 daemon prio=5 os_prio=0 tid=0x00007f82b6653000 nid=0x1f4fa
in Object.wait() [0x00007f8135ff6000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x00000006815a8d40> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"BoneCP-pool-watch-thread" #101 daemon prio=5 os_prio=0
tid=0x00007f82b662a800 nid=0x1f4e8 waiting on condition [0x00007f81368f7000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x000000068189e3b8> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-keep-alive-scheduler" #100 daemon prio=5 os_prio=0
tid=0x00007f82b6628800 nid=0x1f4e7 waiting on condition [0x00007f81369f8000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x000000068189ed50> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"org.spark-project.guava.base.internal.Finalizer" #99 daemon prio=5
os_prio=0 tid=0x00007f82b6627800 nid=0x1f4e6 in Object.wait()
[0x00007f8136af9000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x000000068189f2e8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-pool-watch-thread" #98 daemon prio=5 os_prio=0
tid=0x00007f82b63c0800 nid=0x1f4e5 waiting on condition [0x00007f8136ffa000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681d1a610> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"BoneCP-keep-alive-scheduler" #97 daemon prio=5 os_prio=0
tid=0x00007f82b63bf800 nid=0x1f4e4 waiting on condition [0x00007f81370fb000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000681d1b050> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"org.spark-project.guava.base.internal.Finalizer" #96 daemon prio=5
os_prio=0 tid=0x00007f82b618d000 nid=0x1f4e3 in Object.wait()
[0x00007f81371fc000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x0000000681d1b5e8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
at java.lang.Thread.run(Thread.java:745)

"derby.rawStoreDaemon" #93 daemon prio=5 os_prio=0 tid=0x00007f82b63a6000
nid=0x1f4e1 in Object.wait() [0x00007f81375fe000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
- locked <0x0000000680a3a2f8> (a
org.apache.derby.impl.services.daemon.BasicDaemon)
at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
at java.lang.Thread.run(Thread.java:745)

"Timer-1" #92 daemon prio=5 os_prio=0 tid=0x00007f82b6322800 nid=0x1f4e0 in
Object.wait() [0x00007f814c1ef000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x0000000680a6c4b8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"SparkListenerBus" #20 daemon prio=5 os_prio=0 tid=0x00007f82b5354000
nid=0x1f4df waiting on condition [0x00007f81581f3000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680299370> (a
java.util.concurrent.Semaphore$NonfairSync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
at
org.apache.spark.util.AsynchronousListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(AsynchronousListenerBus.scala:65)
at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
at
org.apache.spark.util.AsynchronousListenerBus$$anon$1.run(AsynchronousListenerBus.scala:63)

"Spark Context Cleaner" #90 daemon prio=5 os_prio=0 tid=0x00007f82b5358000
nid=0x1f4de in Object.wait() [0x00007f81582f4000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x0000000680a496c8> (a java.lang.ref.ReferenceQueue$Lock)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply$mcV$sp(ContextCleaner.scala:157)
at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
at org.apache.spark.ContextCleaner.org
$apache$spark$ContextCleaner$$keepCleaning(ContextCleaner.scala:154)
at org.apache.spark.ContextCleaner$$anon$3.run(ContextCleaner.scala:67)

"shuffle-server-0" #81 daemon prio=5 os_prio=0 tid=0x00007f82b52c8000
nid=0x1f4dd runnable [0x00007f81583f5000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680a678b0> (a
io.netty.channel.nio.SelectedSelectionKeySet)
- locked <0x0000000680a678d0> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680a67868> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
at
io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
at java.lang.Thread.run(Thread.java:745)

"MesosSchedulerBackend-mesos-driver" #71 daemon prio=5 os_prio=0
tid=0x00007f82b4eb4800 nid=0x1f461 runnable [0x00007f824831b000]
   java.lang.Thread.State: RUNNABLE
at org.apache.mesos.MesosSchedulerDriver.join(Native Method)
at org.apache.mesos.MesosSchedulerDriver.run(MesosSchedulerDriver.java:168)
at
org.apache.spark.scheduler.cluster.mesos.MesosSchedulerUtils$$anon$1.run(MesosSchedulerUtils.scala:116)

"qtp2030121719-66" #66 daemon prio=5 os_prio=0 tid=0x00007f817c006800
nid=0x1f45f waiting on condition [0x00007f81587f7000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x000000068076a390> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-69" #69 daemon prio=5 os_prio=0 tid=0x00007f8190001000
nid=0x1f45e waiting on condition [0x00007f8248119000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x000000068076a390> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-65 Acceptor5 SelectChannelConnector@0.0.0.0:4040" #65 daemon
prio=5 os_prio=0 tid=0x00007f8180001000 nid=0x1f45b waiting for monitor
entry [0x00007f824841c000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
- waiting to lock <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"dag-scheduler-event-loop" #64 daemon prio=5 os_prio=0
tid=0x00007f82b4e28800 nid=0x1f441 waiting on condition [0x00007f825a0e3000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680806280> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at
java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
at
java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:46)

"Timer-0" #63 daemon prio=5 os_prio=0 tid=0x00007f82b4df1000 nid=0x1f440 in
Object.wait() [0x00007f825a1e4000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x00000006807432e8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"heartbeat-receiver-event-loop-thread" #62 daemon prio=5 os_prio=0
tid=0x00007f81e40b7800 nid=0x1f43f waiting on condition [0x00007f8259f48000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680852bf0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-61 Acceptor4 SelectChannelConnector@0.0.0.0:4040" #61 daemon
prio=5 os_prio=0 tid=0x00007f82b4d76000 nid=0x1f43e waiting for monitor
entry [0x00007f825a2e5000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
- waiting to lock <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-60 Acceptor1 SelectChannelConnector@0.0.0.0:4040" #60 daemon
prio=5 os_prio=0 tid=0x00007f82b4cdd000 nid=0x1f43d runnable
[0x00007f825a3e6000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
- locked <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-59 Selector5" #59 daemon prio=5 os_prio=0
tid=0x00007f82b4cdc000 nid=0x1f43c runnable [0x00007f825a4e7000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000006807bc128> (a sun.nio.ch.Util$2)
- locked <0x00000006807bc118> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000006807bbff0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-58 Acceptor3 SelectChannelConnector@0.0.0.0:4040" #58 daemon
prio=5 os_prio=0 tid=0x00007f82b4cd9800 nid=0x1f43b waiting for monitor
entry [0x00007f825a5e8000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
- waiting to lock <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-57 Acceptor2 SelectChannelConnector@0.0.0.0:4040" #57 daemon
prio=5 os_prio=0 tid=0x00007f82b4cd7800 nid=0x1f43a waiting for monitor
entry [0x00007f825a6e9000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
- waiting to lock <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-56 Acceptor0 SelectChannelConnector@0.0.0.0:4040" #56 daemon
prio=5 os_prio=0 tid=0x00007f82b4cd6000 nid=0x1f439 waiting for monitor
entry [0x00007f825a7ea000]
   java.lang.Thread.State: BLOCKED (on object monitor)
at
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
- waiting to lock <0x00000006807c0670> (a java.lang.Object)
at
org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-55 Selector4" #55 daemon prio=5 os_prio=0
tid=0x00007f82b4cd4000 nid=0x1f438 runnable [0x00007f825a8eb000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068076ffd8> (a sun.nio.ch.Util$2)
- locked <0x000000068076ffc8> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068076fea0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-54 Selector3" #54 daemon prio=5 os_prio=0
tid=0x00007f82b4cd2800 nid=0x1f437 runnable [0x00007f825a9ec000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068076fa60> (a sun.nio.ch.Util$2)
- locked <0x000000068076fa50> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068076f928> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-53 Selector2" #53 daemon prio=5 os_prio=0
tid=0x00007f82b4cd0800 nid=0x1f436 runnable [0x00007f825aaed000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068076f4e8> (a sun.nio.ch.Util$2)
- locked <0x000000068076f4d8> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068076f3b0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-52 Selector1" #52 daemon prio=5 os_prio=0
tid=0x00007f82b4ccf000 nid=0x1f435 runnable [0x00007f825abee000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068076ef70> (a sun.nio.ch.Util$2)
- locked <0x000000068076ef60> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068076ee38> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"qtp2030121719-51 Selector0" #51 daemon prio=5 os_prio=0
tid=0x00007f82b4ccd800 nid=0x1f434 runnable [0x00007f825acef000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068076a8c8> (a sun.nio.ch.Util$2)
- locked <0x000000068076a8b8> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068076a780> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at
org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"SPARK_CONTEXT cleanup timer" #49 daemon prio=5 os_prio=0
tid=0x00007f82b4c5f800 nid=0x1f432 in Object.wait() [0x00007f825aff0000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x0000000680883998> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x0000000680883998> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"qtp1372137902-48" #48 daemon prio=5 os_prio=0 tid=0x00007f82b4c53000
nid=0x1f431 waiting on condition [0x00007f825b0f1000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-47" #47 daemon prio=5 os_prio=0 tid=0x00007f82b4c51800
nid=0x1f430 waiting on condition [0x00007f825b1f2000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-46" #46 daemon prio=5 os_prio=0 tid=0x00007f82b4c50000
nid=0x1f42f waiting on condition [0x00007f825b2f3000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-44" #44 daemon prio=5 os_prio=0 tid=0x00007f82b4c4c800
nid=0x1f42d waiting on condition [0x00007f825b4f5000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-43" #43 daemon prio=5 os_prio=0 tid=0x00007f82b4c4a800
nid=0x1f42c waiting on condition [0x00007f825b5f6000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-42" #42 daemon prio=5 os_prio=0 tid=0x00007f82b4c49000
nid=0x1f42b waiting on condition [0x00007f825b6f7000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680859028> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)

"qtp1372137902-41 Acceptor0 SocketConnector@0.0.0.0:57827" #41 daemon
prio=5 os_prio=0 tid=0x00007f82b4c48000 nid=0x1f42a runnable
[0x00007f825b7f8000]
   java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
at java.net.ServerSocket.implAccept(ServerSocket.java:545)
at java.net.ServerSocket.accept(ServerSocket.java:513)
at
org.spark-project.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
at
org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at
org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)

"BROADCAST_VARS cleanup timer" #39 daemon prio=5 os_prio=0
tid=0x00007f82b4bb3000 nid=0x1f429 in Object.wait() [0x00007f825b8f9000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000068082bdf8> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x000000068082bdf8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"BLOCK_MANAGER cleanup timer" #38 daemon prio=5 os_prio=0
tid=0x00007f82b4bb1800 nid=0x1f428 in Object.wait() [0x00007f825b9fa000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000068082b8d8> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x000000068082b8d8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"MAP_OUTPUT_TRACKER cleanup timer" #36 daemon prio=5 os_prio=0
tid=0x00007f82b4ae4000 nid=0x1f427 in Object.wait() [0x00007f825bcfb000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x000000068089b5e8> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:502)
at java.util.TimerThread.mainLoop(Timer.java:526)
- locked <0x000000068089b5e8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)

"New I/O server boss #6" #35 daemon prio=5 os_prio=0 tid=0x00007f81e009d000
nid=0x1f426 runnable [0x00007f825bdfc000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680822518> (a sun.nio.ch.Util$2)
- locked <0x0000000680822508> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680822150> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"New I/O worker #5" #34 daemon prio=5 os_prio=0 tid=0x00007f81e009b800
nid=0x1f425 runnable [0x00007f825befd000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000006808217e8> (a sun.nio.ch.Util$2)
- locked <0x00000006808217d8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000006808216c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"New I/O worker #4" #33 daemon prio=5 os_prio=0 tid=0x00007f81e0097800
nid=0x1f424 runnable [0x00007f825bffe000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000068081d200> (a sun.nio.ch.Util$2)
- locked <0x000000068081d1f0> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000068081d0d8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"New I/O boss #3" #32 daemon prio=5 os_prio=0 tid=0x00007f81e009a800
nid=0x1f423 runnable [0x00007f82601b6000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000006808246f0> (a sun.nio.ch.Util$2)
- locked <0x00000006808246e0> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000006808245b8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"New I/O worker #2" #30 daemon prio=5 os_prio=0 tid=0x00007f81e0043000
nid=0x1f422 runnable [0x00007f82602b7000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000006808b7708> (a sun.nio.ch.Util$2)
- locked <0x00000006808b7718> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000006808b76c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"New I/O worker #1" #29 daemon prio=5 os_prio=0 tid=0x00007f81e0026800
nid=0x1f421 runnable [0x00007f82603b8000]
   java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000680883c78> (a sun.nio.ch.Util$2)
- locked <0x0000000680883c88> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000680883c30> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"sparkDriver-akka.remote.default-remote-dispatcher-6" #28 daemon prio=5
os_prio=0 tid=0x00007f82b4a3c800 nid=0x1f420 waiting on condition
[0x00007f82608cb000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680796aa8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at
scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.remote.default-remote-dispatcher-5" #27 daemon prio=5
os_prio=0 tid=0x00007f81e4014000 nid=0x1f41f waiting on condition
[0x00007f82609cc000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x0000000680796aa8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-4" #26 daemon prio=5 os_prio=0
tid=0x00007f81e8001800 nid=0x1f41e waiting on condition [0x00007f8260acd000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-3" #25 daemon prio=5 os_prio=0
tid=0x00007f82b4a11000 nid=0x1f41d waiting on condition [0x00007f8260bce000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-akka.actor.default-dispatcher-2" #24 daemon prio=5 os_prio=0
tid=0x00007f82b4a08000 nid=0x1f41c waiting on condition [0x00007f8260ccf000]
   java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006808c6ad8> (a
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

"sparkDriver-scheduler-1" #23 daemon prio=5 os_prio=0
tid=0x00007f82b4984800 nid=0x1f40d waiting on condition [0x00007f82613d2000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
at
akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
at java.lang.Thread.run(Thread.java:745)

"geofence-updater-0" #18 daemon prio=5 os_prio=0 tid=0x00007f82b421a800
nid=0x1f409 waiting on condition [0x00007f8261cdd000]
   java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for  <0x00000006800260e0> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

"Service Thread" #17 daemon prio=9 os_prio=0 tid=0x00007f82b40ea800
nid=0x1f403 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread11" #16 daemon prio=9 os_prio=0 tid=0x00007f82b40e7800
nid=0x1f402 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread10" #15 daemon prio=9 os_prio=0 tid=0x00007f82b40e5800
nid=0x1f401 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread9" #14 daemon prio=9 os_prio=0 tid=0x00007f82b40e3000
nid=0x1f400 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread8" #13 daemon prio=9 os_prio=0 tid=0x00007f82b40e1000
nid=0x1f3ff waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread7" #12 daemon prio=9 os_prio=0 tid=0x00007f82b40df000
nid=0x1f3fe waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread6" #11 daemon prio=9 os_prio=0 tid=0x00007f82b40dc800
nid=0x1f3fd waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread5" #10 daemon prio=9 os_prio=0 tid=0x00007f82b40da800
nid=0x1f3fc waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread4" #9 daemon prio=9 os_prio=0 tid=0x00007f82b40d0800
nid=0x1f3fb waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007f82b40cc000
nid=0x1f3fa waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007f82b40ca000
nid=0x1f3f9 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007f82b40c8000
nid=0x1f3f8 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007f82b40c5000
nid=0x1f3f7 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f82b40c3000
nid=0x1f3f6 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f82b4099000 nid=0x1f3f5
in Object.wait() [0x00007f8269dfc000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x000000068002e3a8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f82b4097000
nid=0x1f3f4 in Object.wait() [0x00007f8269efd000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
- locked <0x0000000680036080> (a java.lang.ref.Reference$Lock)

"main" #1 prio=5 os_prio=0 tid=0x00007f82b4007800 nid=0x1f3e0 in
Object.wait() [0x00007f82bd479000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.spark.scheduler.JobWaiter.awaitResult(JobWaiter.scala:73)
- locked <0x00000007a9ad7188> (a org.apache.spark.scheduler.JobWaiter)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:559)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1813)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)
at org.apache.spark.rdd.RDD$$anonfun$reduce$1.apply(RDD.scala:1003)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
at org.apache.spark.rdd.RDD.reduce(RDD.scala:985)
at org.apache.spark.rdd.RDD$$anonfun$treeAggregate$1.apply(RDD.scala:1114)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
at org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1091)
at
org.apache.spark.sql.execution.datasources.json.InferSchema$.apply(InferSchema.scala:58)
at
org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:105)
at
org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:100)
at scala.Option.getOrElse(Option.scala:120)
at
org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema$lzycompute(JSONRelation.scala:100)
- locked <0x00000007a9c20f50> (a
org.apache.spark.sql.execution.datasources.json.JSONRelation)
at
org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema(JSONRelation.scala:99)
at
org.apache.spark.sql.sources.HadoopFsRelation.schema$lzycompute(interfaces.scala:561)
- locked <0x00000007a9c20f50> (a
org.apache.spark.sql.execution.datasources.json.JSONRelation)
at
org.apache.spark.sql.sources.HadoopFsRelation.schema(interfaces.scala:560)
at
org.apache.spark.sql.execution.datasources.LogicalRelation.<init>(LogicalRelation.scala:31)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:120)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:104)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:219)
    ...
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:672)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

"VM Thread" os_prio=0 tid=0x00007f82b4092000 nid=0x1f3f3 runnable

"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f82b401d000
nid=0x1f3e1 runnable

"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f82b401f000
nid=0x1f3e2 runnable

"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f82b4020800
nid=0x1f3e3 runnable

"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f82b4022800
nid=0x1f3e4 runnable

"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f82b4024000
nid=0x1f3e5 runnable

"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f82b4026000
nid=0x1f3e6 runnable

"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f82b4027800
nid=0x1f3e7 runnable

"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f82b4029800
nid=0x1f3e8 runnable

"GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007f82b402b000
nid=0x1f3e9 runnable

"GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007f82b402d000
nid=0x1f3ea runnable

"GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007f82b402e800
nid=0x1f3eb runnable

"GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007f82b4030800
nid=0x1f3ec runnable

"GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007f82b4032000
nid=0x1f3ed runnable

"GC task thread#13 (ParallelGC)" os_prio=0 tid=0x00007f82b4034000
nid=0x1f3ee runnable

"GC task thread#14 (ParallelGC)" os_prio=0 tid=0x00007f82b4035800
nid=0x1f3ef runnable

"GC task thread#15 (ParallelGC)" os_prio=0 tid=0x00007f82b4037800
nid=0x1f3f0 runnable

"GC task thread#16 (ParallelGC)" os_prio=0 tid=0x00007f82b4039000
nid=0x1f3f1 runnable

"GC task thread#17 (ParallelGC)" os_prio=0 tid=0x00007f82b403a800
nid=0x1f3f2 runnable

"VM Periodic Task Thread" os_prio=0 tid=0x00007f82b40ed000 nid=0x1f404
waiting on condition

JNI global references: 393

Re: Spark job stuck at DataFrameReader.json() method

Posted by "Bae, Jae Hyeon" <me...@gmail.com>.

Hi Fengdong

Which part of source code do you want to look at? Due to security reason,
copying and pasting whole source code won't be easy.

On Tue, Oct 20, 2015 at 11:56 PM, Fengdong Yu <fe...@everstring.com>
wrote:

> Can you paste your code here?
>
> On Oct 21, 2015, at 2:45 PM, Bae, Jae Hyeon <me...@gmail.com> wrote:
>
> Hi
>
> My Spark job had been running well for a while but since yesterday, it's
> getting stuck on DataFrameReader.json() method. I am guessing that data is
> a little bit changed and InferSchema is having a problem. I am attaching
> thread dump of driver application, can somebody take a look at?
>
> Thank you
> Best, Jae
>
> 2015-10-21 05:54:19
> Full thread dump OpenJDK 64-Bit Server VM (25.45-b02 mixed mode):
>
> "Attach Listener" #1455 daemon prio=9 os_prio=0 tid=0x00007f822c001000
> nid=0x44d waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "shuffle-server-6" #87 daemon prio=5 os_prio=0 tid=0x00007f80c800a000
> nid=0x1f8b8 runnable [0x00007f811c19c000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680ac9858> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680acb8d8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680ac97c0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "shuffle-server-5" #86 daemon prio=5 os_prio=0 tid=0x00007f80c8009000
> nid=0x1f8b2 runnable [0x00007f825b3f4000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680acc050> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680afe028> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680acbfb8> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-411" #411 daemon prio=5 os_prio=0 tid=0x00007f81ac00c000
> nid=0x1f593 waiting on condition [0x00007f811c49d000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "sparkDriver-akka.actor.default-dispatcher-18" #338 daemon prio=5
> os_prio=0 tid=0x00007f81e0273000 nid=0x1f592 waiting on condition
> [0x00007f811c59e000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "task-result-getter-3" #248 daemon prio=5 os_prio=0 tid=0x00007f813c005000
> nid=0x1f591 waiting on condition [0x00007f811c69f000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808fd9b8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "task-result-getter-2" #246 daemon prio=5 os_prio=0 tid=0x00007f8110005000
> nid=0x1f590 waiting on condition [0x00007f811c7a0000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808fd9b8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "task-result-getter-1" #244 daemon prio=5 os_prio=0 tid=0x00007f8144003000
> nid=0x1f58f waiting on condition [0x00007f811c8a1000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808fd9b8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "task-result-getter-0" #242 daemon prio=5 os_prio=0 tid=0x00007f8104006800
> nid=0x1f58e waiting on condition [0x00007f811c9a2000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808fd9b8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "shuffle-server-4" #85 daemon prio=5 os_prio=0 tid=0x00007f80c8007000
> nid=0x1f58d runnable [0x00007f811caa3000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680b027b0> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680b24770> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680b02718> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "shuffle-server-3" #84 daemon prio=5 os_prio=0 tid=0x00007f80c8005800
> nid=0x1f58c runnable [0x00007f811dba5000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680b28ef8> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680b2af78> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680b28e60> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "shuffle-server-2" #83 daemon prio=5 os_prio=0 tid=0x00007f80c8004000
> nid=0x1f58b runnable [0x00007f811dca6000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680b2b6f0> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680b39770> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680b2b658> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "threadDeathWatcher-2-1" #240 daemon prio=1 os_prio=0
> tid=0x00007f8058009000 nid=0x1f58a waiting on condition [0x00007f811eda8000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(Native Method)
> at
> io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:137)
> at
> io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
> at java.lang.Thread.run(Thread.java:745)
>
> "shuffle-server-1" #82 daemon prio=5 os_prio=0 tid=0x00007f80c8003800
> nid=0x1f589 runnable [0x00007f811eea9000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680b3def8> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680b4f1a8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680b3de60> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "sparkDriver-akka.actor.default-dispatcher-17" #148 daemon prio=5
> os_prio=0 tid=0x00007f81e0174800 nid=0x1f516 waiting on condition
> [0x00007f811f1aa000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-16" #147 daemon prio=5
> os_prio=0 tid=0x00007f81dc001000 nid=0x1f515 waiting on condition
> [0x00007f811f2ab000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-15" #146 daemon prio=5
> os_prio=0 tid=0x00007f8074001000 nid=0x1f514 waiting on condition
> [0x00007f811f3ac000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-14" #145 daemon prio=5
> os_prio=0 tid=0x00007f81e8005000 nid=0x1f513 waiting on condition
> [0x00007f81341ee000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "BoneCP-pool-watch-thread" #118 daemon prio=5 os_prio=0
> tid=0x00007f82b5264800 nid=0x1f501 waiting on condition [0x00007f8134aef000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681d60ca8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-keep-alive-scheduler" #117 daemon prio=5 os_prio=0
> tid=0x00007f82b4723800 nid=0x1f500 waiting on condition [0x00007f8134bf0000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681d70538> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "com.google.common.base.internal.Finalizer" #116 daemon prio=5 os_prio=0
> tid=0x00007f82b5268800 nid=0x1f4ff in Object.wait() [0x00007f8134cf1000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x0000000681d964d8> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-pool-watch-thread" #115 daemon prio=5 os_prio=0
> tid=0x00007f82b4ddc800 nid=0x1f4fe waiting on condition [0x00007f81351f2000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681d70798> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-keep-alive-scheduler" #114 daemon prio=5 os_prio=0
> tid=0x00007f82b5924800 nid=0x1f4fd waiting on condition [0x00007f81352f3000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681da64b0> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "com.google.common.base.internal.Finalizer" #113 daemon prio=5 os_prio=0
> tid=0x00007f82b5923000 nid=0x1f4fc in Object.wait() [0x00007f81353f4000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x0000000681d60f38> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
> at java.lang.Thread.run(Thread.java:745)
>
> "derby.rawStoreDaemon" #112 daemon prio=5 os_prio=0 tid=0x00007f82b530d800
> nid=0x1f4fb in Object.wait() [0x00007f81356f5000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
> - locked <0x000000068143fc30> (a
> org.apache.derby.impl.services.daemon.BasicDaemon)
> at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
> at java.lang.Thread.run(Thread.java:745)
>
> "Timer-2" #111 daemon prio=5 os_prio=0 tid=0x00007f82b6653000 nid=0x1f4fa
> in Object.wait() [0x00007f8135ff6000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x00000006815a8d40> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "BoneCP-pool-watch-thread" #101 daemon prio=5 os_prio=0
> tid=0x00007f82b662a800 nid=0x1f4e8 waiting on condition [0x00007f81368f7000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x000000068189e3b8> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-keep-alive-scheduler" #100 daemon prio=5 os_prio=0
> tid=0x00007f82b6628800 nid=0x1f4e7 waiting on condition [0x00007f81369f8000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x000000068189ed50> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "org.spark-project.guava.base.internal.Finalizer" #99 daemon prio=5
> os_prio=0 tid=0x00007f82b6627800 nid=0x1f4e6 in Object.wait()
> [0x00007f8136af9000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x000000068189f2e8> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-pool-watch-thread" #98 daemon prio=5 os_prio=0
> tid=0x00007f82b63c0800 nid=0x1f4e5 waiting on condition [0x00007f8136ffa000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681d1a610> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "BoneCP-keep-alive-scheduler" #97 daemon prio=5 os_prio=0
> tid=0x00007f82b63bf800 nid=0x1f4e4 waiting on condition [0x00007f81370fb000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000681d1b050> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "org.spark-project.guava.base.internal.Finalizer" #96 daemon prio=5
> os_prio=0 tid=0x00007f82b618d000 nid=0x1f4e3 in Object.wait()
> [0x00007f81371fc000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x0000000681d1b5e8> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
> at java.lang.Thread.run(Thread.java:745)
>
> "derby.rawStoreDaemon" #93 daemon prio=5 os_prio=0 tid=0x00007f82b63a6000
> nid=0x1f4e1 in Object.wait() [0x00007f81375fe000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
> - locked <0x0000000680a3a2f8> (a
> org.apache.derby.impl.services.daemon.BasicDaemon)
> at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
> at java.lang.Thread.run(Thread.java:745)
>
> "Timer-1" #92 daemon prio=5 os_prio=0 tid=0x00007f82b6322800 nid=0x1f4e0
> in Object.wait() [0x00007f814c1ef000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x0000000680a6c4b8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "SparkListenerBus" #20 daemon prio=5 os_prio=0 tid=0x00007f82b5354000
> nid=0x1f4df waiting on condition [0x00007f81581f3000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680299370> (a
> java.util.concurrent.Semaphore$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
> at
> org.apache.spark.util.AsynchronousListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(AsynchronousListenerBus.scala:65)
> at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
> at
> org.apache.spark.util.AsynchronousListenerBus$$anon$1.run(AsynchronousListenerBus.scala:63)
>
> "Spark Context Cleaner" #90 daemon prio=5 os_prio=0 tid=0x00007f82b5358000
> nid=0x1f4de in Object.wait() [0x00007f81582f4000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x0000000680a496c8> (a java.lang.ref.ReferenceQueue$Lock)
> at
> org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply$mcV$sp(ContextCleaner.scala:157)
> at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
> at org.apache.spark.ContextCleaner.org
> <http://org.apache.spark.contextcleaner.org/>
> $apache$spark$ContextCleaner$$keepCleaning(ContextCleaner.scala:154)
> at org.apache.spark.ContextCleaner$$anon$3.run(ContextCleaner.scala:67)
>
> "shuffle-server-0" #81 daemon prio=5 os_prio=0 tid=0x00007f82b52c8000
> nid=0x1f4dd runnable [0x00007f81583f5000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680a678b0> (a
> io.netty.channel.nio.SelectedSelectionKeySet)
> - locked <0x0000000680a678d0> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680a67868> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> at
> io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> at java.lang.Thread.run(Thread.java:745)
>
> "MesosSchedulerBackend-mesos-driver" #71 daemon prio=5 os_prio=0
> tid=0x00007f82b4eb4800 nid=0x1f461 runnable [0x00007f824831b000]
>    java.lang.Thread.State: RUNNABLE
> at org.apache.mesos.MesosSchedulerDriver.join(Native Method)
> at org.apache.mesos.MesosSchedulerDriver.run(MesosSchedulerDriver.java:168)
> at
> org.apache.spark.scheduler.cluster.mesos.MesosSchedulerUtils$$anon$1.run(MesosSchedulerUtils.scala:116)
>
> "qtp2030121719-66" #66 daemon prio=5 os_prio=0 tid=0x00007f817c006800
> nid=0x1f45f waiting on condition [0x00007f81587f7000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x000000068076a390> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-69" #69 daemon prio=5 os_prio=0 tid=0x00007f8190001000
> nid=0x1f45e waiting on condition [0x00007f8248119000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x000000068076a390> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-65 Acceptor5 SelectChannelConnector@0.0.0.0:4040" #65
> daemon prio=5 os_prio=0 tid=0x00007f8180001000 nid=0x1f45b waiting for
> monitor entry [0x00007f824841c000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> - waiting to lock <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "dag-scheduler-event-loop" #64 daemon prio=5 os_prio=0
> tid=0x00007f82b4e28800 nid=0x1f441 waiting on condition [0x00007f825a0e3000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680806280> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> at
> java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
> at
> java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
> at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:46)
>
> "Timer-0" #63 daemon prio=5 os_prio=0 tid=0x00007f82b4df1000 nid=0x1f440
> in Object.wait() [0x00007f825a1e4000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x00000006807432e8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "heartbeat-receiver-event-loop-thread" #62 daemon prio=5 os_prio=0
> tid=0x00007f81e40b7800 nid=0x1f43f waiting on condition [0x00007f8259f48000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680852bf0> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-61 Acceptor4 SelectChannelConnector@0.0.0.0:4040" #61
> daemon prio=5 os_prio=0 tid=0x00007f82b4d76000 nid=0x1f43e waiting for
> monitor entry [0x00007f825a2e5000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> - waiting to lock <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-60 Acceptor1 SelectChannelConnector@0.0.0.0:4040" #60
> daemon prio=5 os_prio=0 tid=0x00007f82b4cdd000 nid=0x1f43d runnable
> [0x00007f825a3e6000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
> - locked <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-59 Selector5" #59 daemon prio=5 os_prio=0
> tid=0x00007f82b4cdc000 nid=0x1f43c runnable [0x00007f825a4e7000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x00000006807bc128> (a sun.nio.ch.Util$2)
> - locked <0x00000006807bc118> (a java.util.Collections$UnmodifiableSet)
> - locked <0x00000006807bbff0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-58 Acceptor3 SelectChannelConnector@0.0.0.0:4040" #58
> daemon prio=5 os_prio=0 tid=0x00007f82b4cd9800 nid=0x1f43b waiting for
> monitor entry [0x00007f825a5e8000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> - waiting to lock <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-57 Acceptor2 SelectChannelConnector@0.0.0.0:4040" #57
> daemon prio=5 os_prio=0 tid=0x00007f82b4cd7800 nid=0x1f43a waiting for
> monitor entry [0x00007f825a6e9000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> - waiting to lock <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-56 Acceptor0 SelectChannelConnector@0.0.0.0:4040" #56
> daemon prio=5 os_prio=0 tid=0x00007f82b4cd6000 nid=0x1f439 waiting for
> monitor entry [0x00007f825a7ea000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> at
> sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> - waiting to lock <0x00000006807c0670> (a java.lang.Object)
> at
> org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-55 Selector4" #55 daemon prio=5 os_prio=0
> tid=0x00007f82b4cd4000 nid=0x1f438 runnable [0x00007f825a8eb000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068076ffd8> (a sun.nio.ch.Util$2)
> - locked <0x000000068076ffc8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068076fea0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-54 Selector3" #54 daemon prio=5 os_prio=0
> tid=0x00007f82b4cd2800 nid=0x1f437 runnable [0x00007f825a9ec000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068076fa60> (a sun.nio.ch.Util$2)
> - locked <0x000000068076fa50> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068076f928> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-53 Selector2" #53 daemon prio=5 os_prio=0
> tid=0x00007f82b4cd0800 nid=0x1f436 runnable [0x00007f825aaed000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068076f4e8> (a sun.nio.ch.Util$2)
> - locked <0x000000068076f4d8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068076f3b0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-52 Selector1" #52 daemon prio=5 os_prio=0
> tid=0x00007f82b4ccf000 nid=0x1f435 runnable [0x00007f825abee000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068076ef70> (a sun.nio.ch.Util$2)
> - locked <0x000000068076ef60> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068076ee38> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp2030121719-51 Selector0" #51 daemon prio=5 os_prio=0
> tid=0x00007f82b4ccd800 nid=0x1f434 runnable [0x00007f825acef000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068076a8c8> (a sun.nio.ch.Util$2)
> - locked <0x000000068076a8b8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068076a780> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> at
> org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "SPARK_CONTEXT cleanup timer" #49 daemon prio=5 os_prio=0
> tid=0x00007f82b4c5f800 nid=0x1f432 in Object.wait() [0x00007f825aff0000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x0000000680883998> (a java.util.TaskQueue)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x0000000680883998> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "qtp1372137902-48" #48 daemon prio=5 os_prio=0 tid=0x00007f82b4c53000
> nid=0x1f431 waiting on condition [0x00007f825b0f1000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-47" #47 daemon prio=5 os_prio=0 tid=0x00007f82b4c51800
> nid=0x1f430 waiting on condition [0x00007f825b1f2000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-46" #46 daemon prio=5 os_prio=0 tid=0x00007f82b4c50000
> nid=0x1f42f waiting on condition [0x00007f825b2f3000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-44" #44 daemon prio=5 os_prio=0 tid=0x00007f82b4c4c800
> nid=0x1f42d waiting on condition [0x00007f825b4f5000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-43" #43 daemon prio=5 os_prio=0 tid=0x00007f82b4c4a800
> nid=0x1f42c waiting on condition [0x00007f825b5f6000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-42" #42 daemon prio=5 os_prio=0 tid=0x00007f82b4c49000
> nid=0x1f42b waiting on condition [0x00007f825b6f7000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680859028> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> at java.lang.Thread.run(Thread.java:745)
>
> "qtp1372137902-41 Acceptor0 SocketConnector@0.0.0.0:57827" #41 daemon
> prio=5 os_prio=0 tid=0x00007f82b4c48000 nid=0x1f42a runnable
> [0x00007f825b7f8000]
>    java.lang.Thread.State: RUNNABLE
> at java.net.PlainSocketImpl.socketAccept(Native Method)
> at
> java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
> at java.net.ServerSocket.implAccept(ServerSocket.java:545)
> at java.net.ServerSocket.accept(ServerSocket.java:513)
> at
> org.spark-project.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
> at
> org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at
> org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
> "BROADCAST_VARS cleanup timer" #39 daemon prio=5 os_prio=0
> tid=0x00007f82b4bb3000 nid=0x1f429 in Object.wait() [0x00007f825b8f9000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x000000068082bdf8> (a java.util.TaskQueue)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x000000068082bdf8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "BLOCK_MANAGER cleanup timer" #38 daemon prio=5 os_prio=0
> tid=0x00007f82b4bb1800 nid=0x1f428 in Object.wait() [0x00007f825b9fa000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x000000068082b8d8> (a java.util.TaskQueue)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x000000068082b8d8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "MAP_OUTPUT_TRACKER cleanup timer" #36 daemon prio=5 os_prio=0
> tid=0x00007f82b4ae4000 nid=0x1f427 in Object.wait() [0x00007f825bcfb000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x000000068089b5e8> (a java.util.TaskQueue)
> at java.lang.Object.wait(Object.java:502)
> at java.util.TimerThread.mainLoop(Timer.java:526)
> - locked <0x000000068089b5e8> (a java.util.TaskQueue)
> at java.util.TimerThread.run(Timer.java:505)
>
> "New I/O server boss #6" #35 daemon prio=5 os_prio=0
> tid=0x00007f81e009d000 nid=0x1f426 runnable [0x00007f825bdfc000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680822518> (a sun.nio.ch.Util$2)
> - locked <0x0000000680822508> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680822150> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
> at
> org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #5" #34 daemon prio=5 os_prio=0 tid=0x00007f81e009b800
> nid=0x1f425 runnable [0x00007f825befd000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x00000006808217e8> (a sun.nio.ch.Util$2)
> - locked <0x00000006808217d8> (a java.util.Collections$UnmodifiableSet)
> - locked <0x00000006808216c0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #4" #33 daemon prio=5 os_prio=0 tid=0x00007f81e0097800
> nid=0x1f424 runnable [0x00007f825bffe000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x000000068081d200> (a sun.nio.ch.Util$2)
> - locked <0x000000068081d1f0> (a java.util.Collections$UnmodifiableSet)
> - locked <0x000000068081d0d8> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O boss #3" #32 daemon prio=5 os_prio=0 tid=0x00007f81e009a800
> nid=0x1f423 runnable [0x00007f82601b6000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x00000006808246f0> (a sun.nio.ch.Util$2)
> - locked <0x00000006808246e0> (a java.util.Collections$UnmodifiableSet)
> - locked <0x00000006808245b8> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #2" #30 daemon prio=5 os_prio=0 tid=0x00007f81e0043000
> nid=0x1f422 runnable [0x00007f82602b7000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x00000006808b7708> (a sun.nio.ch.Util$2)
> - locked <0x00000006808b7718> (a java.util.Collections$UnmodifiableSet)
> - locked <0x00000006808b76c0> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "New I/O worker #1" #29 daemon prio=5 os_prio=0 tid=0x00007f81e0026800
> nid=0x1f421 runnable [0x00007f82603b8000]
>    java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> - locked <0x0000000680883c78> (a sun.nio.ch.Util$2)
> - locked <0x0000000680883c88> (a java.util.Collections$UnmodifiableSet)
> - locked <0x0000000680883c30> (a sun.nio.ch.EPollSelectorImpl)
> at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> at
> org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> at
> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> at
> org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "sparkDriver-akka.remote.default-remote-dispatcher-6" #28 daemon prio=5
> os_prio=0 tid=0x00007f82b4a3c800 nid=0x1f420 waiting on condition
> [0x00007f82608cb000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680796aa8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at
> scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.remote.default-remote-dispatcher-5" #27 daemon prio=5
> os_prio=0 tid=0x00007f81e4014000 nid=0x1f41f waiting on condition
> [0x00007f82609cc000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x0000000680796aa8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-4" #26 daemon prio=5 os_prio=0
> tid=0x00007f81e8001800 nid=0x1f41e waiting on condition [0x00007f8260acd000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-3" #25 daemon prio=5 os_prio=0
> tid=0x00007f82b4a11000 nid=0x1f41d waiting on condition [0x00007f8260bce000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-akka.actor.default-dispatcher-2" #24 daemon prio=5 os_prio=0
> tid=0x00007f82b4a08000 nid=0x1f41c waiting on condition [0x00007f8260ccf000]
>    java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006808c6ad8> (a
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>
> "sparkDriver-scheduler-1" #23 daemon prio=5 os_prio=0
> tid=0x00007f82b4984800 nid=0x1f40d waiting on condition [0x00007f82613d2000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> at java.lang.Thread.sleep(Native Method)
> at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
> at
> akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
> at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
> at java.lang.Thread.run(Thread.java:745)
>
> "geofence-updater-0" #18 daemon prio=5 os_prio=0 tid=0x00007f82b421a800
> nid=0x1f409 waiting on condition [0x00007f8261cdd000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for  <0x00000006800260e0> (a
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> at
> java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
>
> "Service Thread" #17 daemon prio=9 os_prio=0 tid=0x00007f82b40ea800
> nid=0x1f403 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C1 CompilerThread11" #16 daemon prio=9 os_prio=0 tid=0x00007f82b40e7800
> nid=0x1f402 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C1 CompilerThread10" #15 daemon prio=9 os_prio=0 tid=0x00007f82b40e5800
> nid=0x1f401 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C1 CompilerThread9" #14 daemon prio=9 os_prio=0 tid=0x00007f82b40e3000
> nid=0x1f400 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C1 CompilerThread8" #13 daemon prio=9 os_prio=0 tid=0x00007f82b40e1000
> nid=0x1f3ff waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread7" #12 daemon prio=9 os_prio=0 tid=0x00007f82b40df000
> nid=0x1f3fe waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread6" #11 daemon prio=9 os_prio=0 tid=0x00007f82b40dc800
> nid=0x1f3fd waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread5" #10 daemon prio=9 os_prio=0 tid=0x00007f82b40da800
> nid=0x1f3fc waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread4" #9 daemon prio=9 os_prio=0 tid=0x00007f82b40d0800
> nid=0x1f3fb waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007f82b40cc000
> nid=0x1f3fa waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007f82b40ca000
> nid=0x1f3f9 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007f82b40c8000
> nid=0x1f3f8 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007f82b40c5000
> nid=0x1f3f7 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f82b40c3000
> nid=0x1f3f6 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
>
> "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f82b4099000 nid=0x1f3f5
> in Object.wait() [0x00007f8269dfc000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> - locked <0x000000068002e3a8> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
>
> "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f82b4097000
> nid=0x1f3f4 in Object.wait() [0x00007f8269efd000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
> - locked <0x0000000680036080> (a java.lang.ref.Reference$Lock)
>
> "main" #1 prio=5 os_prio=0 tid=0x00007f82b4007800 nid=0x1f3e0 in
> Object.wait() [0x00007f82bd479000]
>    java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at org.apache.spark.scheduler.JobWaiter.awaitResult(JobWaiter.scala:73)
> - locked <0x00000007a9ad7188> (a org.apache.spark.scheduler.JobWaiter)
> at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:559)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1813)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)
> at org.apache.spark.rdd.RDD$$anonfun$reduce$1.apply(RDD.scala:1003)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
> at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
> at org.apache.spark.rdd.RDD.reduce(RDD.scala:985)
> at org.apache.spark.rdd.RDD$$anonfun$treeAggregate$1.apply(RDD.scala:1114)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
> at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
> at org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1091)
> at
> org.apache.spark.sql.execution.datasources.json.InferSchema$.apply(InferSchema.scala:58)
> at
> org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:105)
> at
> org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:100)
> at scala.Option.getOrElse(Option.scala:120)
> at
> org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema$lzycompute(JSONRelation.scala:100)
> - locked <0x00000007a9c20f50> (a
> org.apache.spark.sql.execution.datasources.json.JSONRelation)
> at
> org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema(JSONRelation.scala:99)
> at
> org.apache.spark.sql.sources.HadoopFsRelation.schema$lzycompute(interfaces.scala:561)
> - locked <0x00000007a9c20f50> (a
> org.apache.spark.sql.execution.datasources.json.JSONRelation)
> at
> org.apache.spark.sql.sources.HadoopFsRelation.schema(interfaces.scala:560)
> at
> org.apache.spark.sql.execution.datasources.LogicalRelation.<init>(LogicalRelation.scala:31)
> at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:120)
> at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:104)
> at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:219)
>     ...
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:672)
> at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
>
> "VM Thread" os_prio=0 tid=0x00007f82b4092000 nid=0x1f3f3 runnable
>
> "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f82b401d000
> nid=0x1f3e1 runnable
>
> "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f82b401f000
> nid=0x1f3e2 runnable
>
> "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f82b4020800
> nid=0x1f3e3 runnable
>
> "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f82b4022800
> nid=0x1f3e4 runnable
>
> "GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f82b4024000
> nid=0x1f3e5 runnable
>
> "GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f82b4026000
> nid=0x1f3e6 runnable
>
> "GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f82b4027800
> nid=0x1f3e7 runnable
>
> "GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f82b4029800
> nid=0x1f3e8 runnable
>
> "GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007f82b402b000
> nid=0x1f3e9 runnable
>
> "GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007f82b402d000
> nid=0x1f3ea runnable
>
> "GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007f82b402e800
> nid=0x1f3eb runnable
>
> "GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007f82b4030800
> nid=0x1f3ec runnable
>
> "GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007f82b4032000
> nid=0x1f3ed runnable
>
> "GC task thread#13 (ParallelGC)" os_prio=0 tid=0x00007f82b4034000
> nid=0x1f3ee runnable
>
> "GC task thread#14 (ParallelGC)" os_prio=0 tid=0x00007f82b4035800
> nid=0x1f3ef runnable
>
> "GC task thread#15 (ParallelGC)" os_prio=0 tid=0x00007f82b4037800
> nid=0x1f3f0 runnable
>
> "GC task thread#16 (ParallelGC)" os_prio=0 tid=0x00007f82b4039000
> nid=0x1f3f1 runnable
>
> "GC task thread#17 (ParallelGC)" os_prio=0 tid=0x00007f82b403a800
> nid=0x1f3f2 runnable
>
> "VM Periodic Task Thread" os_prio=0 tid=0x00007f82b40ed000 nid=0x1f404
> waiting on condition
>
> JNI global references: 393
>
>
>

Re: Spark job stuck at DataFrameReader.json() method

Posted by Fengdong Yu <fe...@everstring.com>.

Can you paste your code here?

> On Oct 21, 2015, at 2:45 PM, Bae, Jae Hyeon <me...@gmail.com> wrote:
> 
> Hi
> 
> My Spark job had been running well for a while but since yesterday, it's getting stuck on DataFrameReader.json() method. I am guessing that data is a little bit changed and InferSchema is having a problem. I am attaching thread dump of driver application, can somebody take a look at?
> 
> Thank you
> Best, Jae
> 
> 2015-10-21 05:54:19
> Full thread dump OpenJDK 64-Bit Server VM (25.45-b02 mixed mode):
> 
> "Attach Listener" #1455 daemon prio=9 os_prio=0 tid=0x00007f822c001000 nid=0x44d waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "shuffle-server-6" #87 daemon prio=5 os_prio=0 tid=0x00007f80c800a000 nid=0x1f8b8 runnable [0x00007f811c19c000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680ac9858> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680acb8d8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680ac97c0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "shuffle-server-5" #86 daemon prio=5 os_prio=0 tid=0x00007f80c8009000 nid=0x1f8b2 runnable [0x00007f825b3f4000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680acc050> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680afe028> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680acbfb8> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-411" #411 daemon prio=5 os_prio=0 tid=0x00007f81ac00c000 nid=0x1f593 waiting on condition [0x00007f811c49d000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "sparkDriver-akka.actor.default-dispatcher-18" #338 daemon prio=5 os_prio=0 tid=0x00007f81e0273000 nid=0x1f592 waiting on condition [0x00007f811c59e000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "task-result-getter-3" #248 daemon prio=5 os_prio=0 tid=0x00007f813c005000 nid=0x1f591 waiting on condition [0x00007f811c69f000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808fd9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "task-result-getter-2" #246 daemon prio=5 os_prio=0 tid=0x00007f8110005000 nid=0x1f590 waiting on condition [0x00007f811c7a0000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808fd9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "task-result-getter-1" #244 daemon prio=5 os_prio=0 tid=0x00007f8144003000 nid=0x1f58f waiting on condition [0x00007f811c8a1000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808fd9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "task-result-getter-0" #242 daemon prio=5 os_prio=0 tid=0x00007f8104006800 nid=0x1f58e waiting on condition [0x00007f811c9a2000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808fd9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "shuffle-server-4" #85 daemon prio=5 os_prio=0 tid=0x00007f80c8007000 nid=0x1f58d runnable [0x00007f811caa3000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680b027b0> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680b24770> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680b02718> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "shuffle-server-3" #84 daemon prio=5 os_prio=0 tid=0x00007f80c8005800 nid=0x1f58c runnable [0x00007f811dba5000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680b28ef8> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680b2af78> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680b28e60> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "shuffle-server-2" #83 daemon prio=5 os_prio=0 tid=0x00007f80c8004000 nid=0x1f58b runnable [0x00007f811dca6000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680b2b6f0> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680b39770> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680b2b658> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "threadDeathWatcher-2-1" #240 daemon prio=1 os_prio=0 tid=0x00007f8058009000 nid=0x1f58a waiting on condition [0x00007f811eda8000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:137)
> 	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "shuffle-server-1" #82 daemon prio=5 os_prio=0 tid=0x00007f80c8003800 nid=0x1f589 runnable [0x00007f811eea9000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680b3def8> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680b4f1a8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680b3de60> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "sparkDriver-akka.actor.default-dispatcher-17" #148 daemon prio=5 os_prio=0 tid=0x00007f81e0174800 nid=0x1f516 waiting on condition [0x00007f811f1aa000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-16" #147 daemon prio=5 os_prio=0 tid=0x00007f81dc001000 nid=0x1f515 waiting on condition [0x00007f811f2ab000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-15" #146 daemon prio=5 os_prio=0 tid=0x00007f8074001000 nid=0x1f514 waiting on condition [0x00007f811f3ac000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-14" #145 daemon prio=5 os_prio=0 tid=0x00007f81e8005000 nid=0x1f513 waiting on condition [0x00007f81341ee000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "BoneCP-pool-watch-thread" #118 daemon prio=5 os_prio=0 tid=0x00007f82b5264800 nid=0x1f501 waiting on condition [0x00007f8134aef000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681d60ca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 	at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-keep-alive-scheduler" #117 daemon prio=5 os_prio=0 tid=0x00007f82b4723800 nid=0x1f500 waiting on condition [0x00007f8134bf0000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681d70538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "com.google.common.base.internal.Finalizer" #116 daemon prio=5 os_prio=0 tid=0x00007f82b5268800 nid=0x1f4ff in Object.wait() [0x00007f8134cf1000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x0000000681d964d8> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-pool-watch-thread" #115 daemon prio=5 os_prio=0 tid=0x00007f82b4ddc800 nid=0x1f4fe waiting on condition [0x00007f81351f2000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681d70798> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 	at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-keep-alive-scheduler" #114 daemon prio=5 os_prio=0 tid=0x00007f82b5924800 nid=0x1f4fd waiting on condition [0x00007f81352f3000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681da64b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "com.google.common.base.internal.Finalizer" #113 daemon prio=5 os_prio=0 tid=0x00007f82b5923000 nid=0x1f4fc in Object.wait() [0x00007f81353f4000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x0000000681d60f38> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at com.google.common.base.internal.Finalizer.run(Finalizer.java:131)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "derby.rawStoreDaemon" #112 daemon prio=5 os_prio=0 tid=0x00007f82b530d800 nid=0x1f4fb in Object.wait() [0x00007f81356f5000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
> 	- locked <0x000000068143fc30> (a org.apache.derby.impl.services.daemon.BasicDaemon)
> 	at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "Timer-2" #111 daemon prio=5 os_prio=0 tid=0x00007f82b6653000 nid=0x1f4fa in Object.wait() [0x00007f8135ff6000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x00000006815a8d40> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "BoneCP-pool-watch-thread" #101 daemon prio=5 os_prio=0 tid=0x00007f82b662a800 nid=0x1f4e8 waiting on condition [0x00007f81368f7000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x000000068189e3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 	at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-keep-alive-scheduler" #100 daemon prio=5 os_prio=0 tid=0x00007f82b6628800 nid=0x1f4e7 waiting on condition [0x00007f81369f8000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x000000068189ed50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "org.spark-project.guava.base.internal.Finalizer" #99 daemon prio=5 os_prio=0 tid=0x00007f82b6627800 nid=0x1f4e6 in Object.wait() [0x00007f8136af9000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x000000068189f2e8> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-pool-watch-thread" #98 daemon prio=5 os_prio=0 tid=0x00007f82b63c0800 nid=0x1f4e5 waiting on condition [0x00007f8136ffa000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681d1a610> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
> 	at com.jolbox.bonecp.PoolWatchThread.run(PoolWatchThread.java:75)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BoneCP-keep-alive-scheduler" #97 daemon prio=5 os_prio=0 tid=0x00007f82b63bf800 nid=0x1f4e4 waiting on condition [0x00007f81370fb000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000681d1b050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "org.spark-project.guava.base.internal.Finalizer" #96 daemon prio=5 os_prio=0 tid=0x00007f82b618d000 nid=0x1f4e3 in Object.wait() [0x00007f81371fc000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x0000000681d1b5e8> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at org.spark-project.guava.base.internal.Finalizer.run(Finalizer.java:132)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "derby.rawStoreDaemon" #93 daemon prio=5 os_prio=0 tid=0x00007f82b63a6000 nid=0x1f4e1 in Object.wait() [0x00007f81375fe000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
> 	- locked <0x0000000680a3a2f8> (a org.apache.derby.impl.services.daemon.BasicDaemon)
> 	at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "Timer-1" #92 daemon prio=5 os_prio=0 tid=0x00007f82b6322800 nid=0x1f4e0 in Object.wait() [0x00007f814c1ef000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x0000000680a6c4b8> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "SparkListenerBus" #20 daemon prio=5 os_prio=0 tid=0x00007f82b5354000 nid=0x1f4df waiting on condition [0x00007f81581f3000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680299370> (a java.util.concurrent.Semaphore$NonfairSync)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> 	at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
> 	at org.apache.spark.util.AsynchronousListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(AsynchronousListenerBus.scala:65)
> 	at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
> 	at org.apache.spark.util.AsynchronousListenerBus$$anon$1.run(AsynchronousListenerBus.scala:63)
> 
> "Spark Context Cleaner" #90 daemon prio=5 os_prio=0 tid=0x00007f82b5358000 nid=0x1f4de in Object.wait() [0x00007f81582f4000]
>    java.lang.Thread.State: TIMED_WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x0000000680a496c8> (a java.lang.ref.ReferenceQueue$Lock)
> 	at org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply$mcV$sp(ContextCleaner.scala:157)
> 	at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)
> 	at org.apache.spark.ContextCleaner.org <http://org.apache.spark.contextcleaner.org/>$apache$spark$ContextCleaner$$keepCleaning(ContextCleaner.scala:154)
> 	at org.apache.spark.ContextCleaner$$anon$3.run(ContextCleaner.scala:67)
> 
> "shuffle-server-0" #81 daemon prio=5 os_prio=0 tid=0x00007f82b52c8000 nid=0x1f4dd runnable [0x00007f81583f5000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680a678b0> (a io.netty.channel.nio.SelectedSelectionKeySet)
> 	- locked <0x0000000680a678d0> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680a67868> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:622)
> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:310)
> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "MesosSchedulerBackend-mesos-driver" #71 daemon prio=5 os_prio=0 tid=0x00007f82b4eb4800 nid=0x1f461 runnable [0x00007f824831b000]
>    java.lang.Thread.State: RUNNABLE
> 	at org.apache.mesos.MesosSchedulerDriver.join(Native Method)
> 	at org.apache.mesos.MesosSchedulerDriver.run(MesosSchedulerDriver.java:168)
> 	at org.apache.spark.scheduler.cluster.mesos.MesosSchedulerUtils$$anon$1.run(MesosSchedulerUtils.scala:116)
> 
> "qtp2030121719-66" #66 daemon prio=5 os_prio=0 tid=0x00007f817c006800 nid=0x1f45f waiting on condition [0x00007f81587f7000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x000000068076a390> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-69" #69 daemon prio=5 os_prio=0 tid=0x00007f8190001000 nid=0x1f45e waiting on condition [0x00007f8248119000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x000000068076a390> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-65 Acceptor5 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #65 daemon prio=5 os_prio=0 tid=0x00007f8180001000 nid=0x1f45b waiting for monitor entry [0x00007f824841c000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> 	- waiting to lock <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "dag-scheduler-event-loop" #64 daemon prio=5 os_prio=0 tid=0x00007f82b4e28800 nid=0x1f441 waiting on condition [0x00007f825a0e3000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680806280> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
> 	at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
> 	at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
> 	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:46)
> 
> "Timer-0" #63 daemon prio=5 os_prio=0 tid=0x00007f82b4df1000 nid=0x1f440 in Object.wait() [0x00007f825a1e4000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x00000006807432e8> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "heartbeat-receiver-event-loop-thread" #62 daemon prio=5 os_prio=0 tid=0x00007f81e40b7800 nid=0x1f43f waiting on condition [0x00007f8259f48000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680852bf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-61 Acceptor4 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #61 daemon prio=5 os_prio=0 tid=0x00007f82b4d76000 nid=0x1f43e waiting for monitor entry [0x00007f825a2e5000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> 	- waiting to lock <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-60 Acceptor1 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #60 daemon prio=5 os_prio=0 tid=0x00007f82b4cdd000 nid=0x1f43d runnable [0x00007f825a3e6000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
> 	- locked <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-59 Selector5" #59 daemon prio=5 os_prio=0 tid=0x00007f82b4cdc000 nid=0x1f43c runnable [0x00007f825a4e7000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x00000006807bc128> (a sun.nio.ch.Util$2)
> 	- locked <0x00000006807bc118> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x00000006807bbff0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-58 Acceptor3 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #58 daemon prio=5 os_prio=0 tid=0x00007f82b4cd9800 nid=0x1f43b waiting for monitor entry [0x00007f825a5e8000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> 	- waiting to lock <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-57 Acceptor2 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #57 daemon prio=5 os_prio=0 tid=0x00007f82b4cd7800 nid=0x1f43a waiting for monitor entry [0x00007f825a6e9000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> 	- waiting to lock <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-56 Acceptor0 SelectChannelConnector@0.0.0.0:4040 <http://SelectChannelConnector@0.0.0.0:4040/>" #56 daemon prio=5 os_prio=0 tid=0x00007f82b4cd6000 nid=0x1f439 waiting for monitor entry [0x00007f825a7ea000]
>    java.lang.Thread.State: BLOCKED (on object monitor)
> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:234)
> 	- waiting to lock <0x00000006807c0670> (a java.lang.Object)
> 	at org.spark-project.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-55 Selector4" #55 daemon prio=5 os_prio=0 tid=0x00007f82b4cd4000 nid=0x1f438 runnable [0x00007f825a8eb000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068076ffd8> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068076ffc8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068076fea0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-54 Selector3" #54 daemon prio=5 os_prio=0 tid=0x00007f82b4cd2800 nid=0x1f437 runnable [0x00007f825a9ec000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068076fa60> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068076fa50> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068076f928> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-53 Selector2" #53 daemon prio=5 os_prio=0 tid=0x00007f82b4cd0800 nid=0x1f436 runnable [0x00007f825aaed000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068076f4e8> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068076f4d8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068076f3b0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-52 Selector1" #52 daemon prio=5 os_prio=0 tid=0x00007f82b4ccf000 nid=0x1f435 runnable [0x00007f825abee000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068076ef70> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068076ef60> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068076ee38> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp2030121719-51 Selector0" #51 daemon prio=5 os_prio=0 tid=0x00007f82b4ccd800 nid=0x1f434 runnable [0x00007f825acef000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068076a8c8> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068076a8b8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068076a780> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.spark-project.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
> 	at org.spark-project.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "SPARK_CONTEXT cleanup timer" #49 daemon prio=5 os_prio=0 tid=0x00007f82b4c5f800 nid=0x1f432 in Object.wait() [0x00007f825aff0000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x0000000680883998> (a java.util.TaskQueue)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x0000000680883998> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "qtp1372137902-48" #48 daemon prio=5 os_prio=0 tid=0x00007f82b4c53000 nid=0x1f431 waiting on condition [0x00007f825b0f1000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-47" #47 daemon prio=5 os_prio=0 tid=0x00007f82b4c51800 nid=0x1f430 waiting on condition [0x00007f825b1f2000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-46" #46 daemon prio=5 os_prio=0 tid=0x00007f82b4c50000 nid=0x1f42f waiting on condition [0x00007f825b2f3000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-44" #44 daemon prio=5 os_prio=0 tid=0x00007f82b4c4c800 nid=0x1f42d waiting on condition [0x00007f825b4f5000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-43" #43 daemon prio=5 os_prio=0 tid=0x00007f82b4c4a800 nid=0x1f42c waiting on condition [0x00007f825b5f6000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-42" #42 daemon prio=5 os_prio=0 tid=0x00007f82b4c49000 nid=0x1f42b waiting on condition [0x00007f825b6f7000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680859028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at org.spark-project.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "qtp1372137902-41 Acceptor0 SocketConnector@0.0.0.0:57827 <http://SocketConnector@0.0.0.0:57827/>" #41 daemon prio=5 os_prio=0 tid=0x00007f82b4c48000 nid=0x1f42a runnable [0x00007f825b7f8000]
>    java.lang.Thread.State: RUNNABLE
> 	at java.net.PlainSocketImpl.socketAccept(Native Method)
> 	at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
> 	at java.net.ServerSocket.implAccept(ServerSocket.java:545)
> 	at java.net.ServerSocket.accept(ServerSocket.java:513)
> 	at org.spark-project.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
> 	at org.spark-project.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> 	at org.spark-project.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "BROADCAST_VARS cleanup timer" #39 daemon prio=5 os_prio=0 tid=0x00007f82b4bb3000 nid=0x1f429 in Object.wait() [0x00007f825b8f9000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x000000068082bdf8> (a java.util.TaskQueue)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x000000068082bdf8> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "BLOCK_MANAGER cleanup timer" #38 daemon prio=5 os_prio=0 tid=0x00007f82b4bb1800 nid=0x1f428 in Object.wait() [0x00007f825b9fa000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x000000068082b8d8> (a java.util.TaskQueue)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x000000068082b8d8> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "MAP_OUTPUT_TRACKER cleanup timer" #36 daemon prio=5 os_prio=0 tid=0x00007f82b4ae4000 nid=0x1f427 in Object.wait() [0x00007f825bcfb000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	- waiting on <0x000000068089b5e8> (a java.util.TaskQueue)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.util.TimerThread.mainLoop(Timer.java:526)
> 	- locked <0x000000068089b5e8> (a java.util.TaskQueue)
> 	at java.util.TimerThread.run(Timer.java:505)
> 
> "New I/O server boss #6" #35 daemon prio=5 os_prio=0 tid=0x00007f81e009d000 nid=0x1f426 runnable [0x00007f825bdfc000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680822518> (a sun.nio.ch.Util$2)
> 	- locked <0x0000000680822508> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680822150> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
> 	at org.jboss.netty.channel.socket.nio.NioServerBoss.select(NioServerBoss.java:163)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.NioServerBoss.run(NioServerBoss.java:42)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "New I/O worker #5" #34 daemon prio=5 os_prio=0 tid=0x00007f81e009b800 nid=0x1f425 runnable [0x00007f825befd000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x00000006808217e8> (a sun.nio.ch.Util$2)
> 	- locked <0x00000006808217d8> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x00000006808216c0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> 	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "New I/O worker #4" #33 daemon prio=5 os_prio=0 tid=0x00007f81e0097800 nid=0x1f424 runnable [0x00007f825bffe000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x000000068081d200> (a sun.nio.ch.Util$2)
> 	- locked <0x000000068081d1f0> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x000000068081d0d8> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> 	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "New I/O boss #3" #32 daemon prio=5 os_prio=0 tid=0x00007f81e009a800 nid=0x1f423 runnable [0x00007f82601b6000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x00000006808246f0> (a sun.nio.ch.Util$2)
> 	- locked <0x00000006808246e0> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x00000006808245b8> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "New I/O worker #2" #30 daemon prio=5 os_prio=0 tid=0x00007f81e0043000 nid=0x1f422 runnable [0x00007f82602b7000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x00000006808b7708> (a sun.nio.ch.Util$2)
> 	- locked <0x00000006808b7718> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x00000006808b76c0> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> 	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "New I/O worker #1" #29 daemon prio=5 os_prio=0 tid=0x00007f81e0026800 nid=0x1f421 runnable [0x00007f82603b8000]
>    java.lang.Thread.State: RUNNABLE
> 	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> 	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
> 	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
> 	- locked <0x0000000680883c78> (a sun.nio.ch.Util$2)
> 	- locked <0x0000000680883c88> (a java.util.Collections$UnmodifiableSet)
> 	- locked <0x0000000680883c30> (a sun.nio.ch.EPollSelectorImpl)
> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
> 	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:415)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
> 	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
> 	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
> 	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> 	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "sparkDriver-akka.remote.default-remote-dispatcher-6" #28 daemon prio=5 os_prio=0 tid=0x00007f82b4a3c800 nid=0x1f420 waiting on condition [0x00007f82608cb000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680796aa8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.idleAwaitWork(ForkJoinPool.java:2135)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2067)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.remote.default-remote-dispatcher-5" #27 daemon prio=5 os_prio=0 tid=0x00007f81e4014000 nid=0x1f41f waiting on condition [0x00007f82609cc000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x0000000680796aa8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-4" #26 daemon prio=5 os_prio=0 tid=0x00007f81e8001800 nid=0x1f41e waiting on condition [0x00007f8260acd000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-3" #25 daemon prio=5 os_prio=0 tid=0x00007f82b4a11000 nid=0x1f41d waiting on condition [0x00007f8260bce000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-akka.actor.default-dispatcher-2" #24 daemon prio=5 os_prio=0 tid=0x00007f82b4a08000 nid=0x1f41c waiting on condition [0x00007f8260ccf000]
>    java.lang.Thread.State: WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006808c6ad8> (a akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinPool)
> 	at scala.concurrent.forkjoin.ForkJoinPool.scan(ForkJoinPool.java:2075)
> 	at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> 	at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> 
> "sparkDriver-scheduler-1" #23 daemon prio=5 os_prio=0 tid=0x00007f82b4984800 nid=0x1f40d waiting on condition [0x00007f82613d2000]
>    java.lang.Thread.State: TIMED_WAITING (sleeping)
> 	at java.lang.Thread.sleep(Native Method)
> 	at akka.actor.LightArrayRevolverScheduler.waitNanos(Scheduler.scala:226)
> 	at akka.actor.LightArrayRevolverScheduler$$anon$8.nextTick(Scheduler.scala:405)
> 	at akka.actor.LightArrayRevolverScheduler$$anon$8.run(Scheduler.scala:375)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "geofence-updater-0" #18 daemon prio=5 os_prio=0 tid=0x00007f82b421a800 nid=0x1f409 waiting on condition [0x00007f8261cdd000]
>    java.lang.Thread.State: TIMED_WAITING (parking)
> 	at sun.misc.Unsafe.park(Native Method)
> 	- parking to wait for  <0x00000006800260e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> "Service Thread" #17 daemon prio=9 os_prio=0 tid=0x00007f82b40ea800 nid=0x1f403 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C1 CompilerThread11" #16 daemon prio=9 os_prio=0 tid=0x00007f82b40e7800 nid=0x1f402 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C1 CompilerThread10" #15 daemon prio=9 os_prio=0 tid=0x00007f82b40e5800 nid=0x1f401 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C1 CompilerThread9" #14 daemon prio=9 os_prio=0 tid=0x00007f82b40e3000 nid=0x1f400 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C1 CompilerThread8" #13 daemon prio=9 os_prio=0 tid=0x00007f82b40e1000 nid=0x1f3ff waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread7" #12 daemon prio=9 os_prio=0 tid=0x00007f82b40df000 nid=0x1f3fe waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread6" #11 daemon prio=9 os_prio=0 tid=0x00007f82b40dc800 nid=0x1f3fd waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread5" #10 daemon prio=9 os_prio=0 tid=0x00007f82b40da800 nid=0x1f3fc waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread4" #9 daemon prio=9 os_prio=0 tid=0x00007f82b40d0800 nid=0x1f3fb waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007f82b40cc000 nid=0x1f3fa waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007f82b40ca000 nid=0x1f3f9 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007f82b40c8000 nid=0x1f3f8 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007f82b40c5000 nid=0x1f3f7 waiting on condition [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007f82b40c3000 nid=0x1f3f6 runnable [0x0000000000000000]
>    java.lang.Thread.State: RUNNABLE
> 
> "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f82b4099000 nid=0x1f3f5 in Object.wait() [0x00007f8269dfc000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
> 	- locked <0x000000068002e3a8> (a java.lang.ref.ReferenceQueue$Lock)
> 	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
> 	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
> 
> "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f82b4097000 nid=0x1f3f4 in Object.wait() [0x00007f8269efd000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.Object.wait(Object.java:502)
> 	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
> 	- locked <0x0000000680036080> (a java.lang.ref.Reference$Lock)
> 
> "main" #1 prio=5 os_prio=0 tid=0x00007f82b4007800 nid=0x1f3e0 in Object.wait() [0x00007f82bd479000]
>    java.lang.Thread.State: WAITING (on object monitor)
> 	at java.lang.Object.wait(Native Method)
> 	at java.lang.Object.wait(Object.java:502)
> 	at org.apache.spark.scheduler.JobWaiter.awaitResult(JobWaiter.scala:73)
> 	- locked <0x00000007a9ad7188> (a org.apache.spark.scheduler.JobWaiter)
> 	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:559)
> 	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1813)
> 	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)
> 	at org.apache.spark.rdd.RDD$$anonfun$reduce$1.apply(RDD.scala:1003)
> 	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
> 	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
> 	at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
> 	at org.apache.spark.rdd.RDD.reduce(RDD.scala:985)
> 	at org.apache.spark.rdd.RDD$$anonfun$treeAggregate$1.apply(RDD.scala:1114)
> 	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
> 	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
> 	at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
> 	at org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1091)
> 	at org.apache.spark.sql.execution.datasources.json.InferSchema$.apply(InferSchema.scala:58)
> 	at org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:105)
> 	at org.apache.spark.sql.execution.datasources.json.JSONRelation$$anonfun$6.apply(JSONRelation.scala:100)
> 	at scala.Option.getOrElse(Option.scala:120)
> 	at org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema$lzycompute(JSONRelation.scala:100)
> 	- locked <0x00000007a9c20f50> (a org.apache.spark.sql.execution.datasources.json.JSONRelation)
> 	at org.apache.spark.sql.execution.datasources.json.JSONRelation.dataSchema(JSONRelation.scala:99)
> 	at org.apache.spark.sql.sources.HadoopFsRelation.schema$lzycompute(interfaces.scala:561)
> 	- locked <0x00000007a9c20f50> (a org.apache.spark.sql.execution.datasources.json.JSONRelation)
> 	at org.apache.spark.sql.sources.HadoopFsRelation.schema(interfaces.scala:560)
> 	at org.apache.spark.sql.execution.datasources.LogicalRelation.<init>(LogicalRelation.scala:31)
> 	at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:120)
> 	at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:104)
> 	at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:219)
>     ...
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:497)
> 	at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:672)
> 	at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> 	at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> 	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120)
> 	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 
> "VM Thread" os_prio=0 tid=0x00007f82b4092000 nid=0x1f3f3 runnable 
> 
> "GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007f82b401d000 nid=0x1f3e1 runnable 
> 
> "GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007f82b401f000 nid=0x1f3e2 runnable 
> 
> "GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007f82b4020800 nid=0x1f3e3 runnable 
> 
> "GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007f82b4022800 nid=0x1f3e4 runnable 
> 
> "GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007f82b4024000 nid=0x1f3e5 runnable 
> 
> "GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007f82b4026000 nid=0x1f3e6 runnable 
> 
> "GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007f82b4027800 nid=0x1f3e7 runnable 
> 
> "GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007f82b4029800 nid=0x1f3e8 runnable 
> 
> "GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007f82b402b000 nid=0x1f3e9 runnable 
> 
> "GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007f82b402d000 nid=0x1f3ea runnable 
> 
> "GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007f82b402e800 nid=0x1f3eb runnable 
> 
> "GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007f82b4030800 nid=0x1f3ec runnable 
> 
> "GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007f82b4032000 nid=0x1f3ed runnable 
> 
> "GC task thread#13 (ParallelGC)" os_prio=0 tid=0x00007f82b4034000 nid=0x1f3ee runnable 
> 
> "GC task thread#14 (ParallelGC)" os_prio=0 tid=0x00007f82b4035800 nid=0x1f3ef runnable 
> 
> "GC task thread#15 (ParallelGC)" os_prio=0 tid=0x00007f82b4037800 nid=0x1f3f0 runnable 
> 
> "GC task thread#16 (ParallelGC)" os_prio=0 tid=0x00007f82b4039000 nid=0x1f3f1 runnable 
> 
> "GC task thread#17 (ParallelGC)" os_prio=0 tid=0x00007f82b403a800 nid=0x1f3f2 runnable 
> 
> "VM Periodic Task Thread" os_prio=0 tid=0x00007f82b40ed000 nid=0x1f404 waiting on condition 
> 
> JNI global references: 393
>