You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/11/22 19:27:15 UTC
[JENKINS] Lucene-Solr-4.10-Linux (32bit/jdk1.7.0_67) - Build # 112
- Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.10-Linux/112/
Java: 32bit/jdk1.7.0_67 -client -XX:+UseConcMarkSweepGC (asserts: true)
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
Error Message:
ERROR: SolrIndexSearcher opens=4 closes=1
Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=4 closes=1
at __randomizedtesting.SeedInfo.seed([4A09753575938C19]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:426)
at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:185)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
Error Message:
45 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest: 1) Thread[id=177, name=qtp3977146-177 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=202, name=qtp4252568-202, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=222, name=qtp9125177-222, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=198, name=qtp4252568-198 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=224, name=qtp9125177-224, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=182, name=qtp3977146-182, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=199, name=qtp4252568-199 Acceptor0 SelectChannelConnector@127.0.0.1:39018, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=214, name=coreZkRegister-288-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=201, name=qtp4252568-201, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=234, name=coreZkRegister-297-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=242, name=OverseerCollectionProcessor-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=WAITING, group=Overseer collection creation process.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307) at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304) at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:514) at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:272) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=223, name=qtp9125177-223, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=188, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 14) Thread[id=243, name=OverseerHdfsCoreFailoverThread-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:136) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=232, name=Thread-89, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 16) Thread[id=194, name=coreZkRegister-279-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=180, name=qtp3977146-180 Acceptor1 SelectChannelConnector@127.0.0.1:40891, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=219, name=qtp9125177-219 Acceptor0 SelectChannelConnector@127.0.0.1:51822, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=228, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 20) Thread[id=217, name=qtp9125177-217 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=220, name=qtp9125177-220 Acceptor1 SelectChannelConnector@127.0.0.1:51822, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=181, name=qtp3977146-181, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=179, name=qtp3977146-179 Acceptor0 SelectChannelConnector@127.0.0.1:40891, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=196, name=HashSessionScavenger-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 25) Thread[id=212, name=Thread-78, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 26) Thread[id=176, name=HashSessionScavenger-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 27) Thread[id=231, name=searcherExecutor-302-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=218, name=qtp9125177-218 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=192, name=Thread-67, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 30) Thread[id=204, name=qtp4252568-204, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=178, name=qtp3977146-178 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=187, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 33) Thread[id=207, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 34) Thread[id=208, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) 35) Thread[id=203, name=qtp4252568-203, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=216, name=HashSessionScavenger-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 37) Thread[id=200, name=qtp4252568-200 Acceptor1 SelectChannelConnector@127.0.0.1:39018, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225) at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=221, name=qtp9125177-221, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=227, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 40) Thread[id=183, name=qtp3977146-183, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=197, name=qtp4252568-197 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=184, name=qtp3977146-184, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=211, name=searcherExecutor-293-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=191, name=searcherExecutor-284-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=241, name=OverseerStateUpdate-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=WAITING, group=Overseer state updater.] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307) at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:604) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:209) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 45 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest:
1) Thread[id=177, name=qtp3977146-177 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=202, name=qtp4252568-202, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=222, name=qtp9125177-222, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=198, name=qtp4252568-198 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=224, name=qtp9125177-224, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=182, name=qtp3977146-182, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=199, name=qtp4252568-199 Acceptor0 SelectChannelConnector@127.0.0.1:39018, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=214, name=coreZkRegister-288-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
9) Thread[id=201, name=qtp4252568-201, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
10) Thread[id=234, name=coreZkRegister-297-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
11) Thread[id=242, name=OverseerCollectionProcessor-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=WAITING, group=Overseer collection creation process.]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:514)
at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:272)
at java.lang.Thread.run(Thread.java:745)
12) Thread[id=223, name=qtp9125177-223, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
13) Thread[id=188, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
14) Thread[id=243, name=OverseerHdfsCoreFailoverThread-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:136)
at java.lang.Thread.run(Thread.java:745)
15) Thread[id=232, name=Thread-89, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
16) Thread[id=194, name=coreZkRegister-279-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
17) Thread[id=180, name=qtp3977146-180 Acceptor1 SelectChannelConnector@127.0.0.1:40891, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
18) Thread[id=219, name=qtp9125177-219 Acceptor0 SelectChannelConnector@127.0.0.1:51822, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
19) Thread[id=228, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
20) Thread[id=217, name=qtp9125177-217 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
21) Thread[id=220, name=qtp9125177-220 Acceptor1 SelectChannelConnector@127.0.0.1:51822, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
22) Thread[id=181, name=qtp3977146-181, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
23) Thread[id=179, name=qtp3977146-179 Acceptor0 SelectChannelConnector@127.0.0.1:40891, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
24) Thread[id=196, name=HashSessionScavenger-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
25) Thread[id=212, name=Thread-78, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
26) Thread[id=176, name=HashSessionScavenger-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
27) Thread[id=231, name=searcherExecutor-302-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
28) Thread[id=218, name=qtp9125177-218 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
29) Thread[id=192, name=Thread-67, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
30) Thread[id=204, name=qtp4252568-204, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
31) Thread[id=178, name=qtp3977146-178 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
32) Thread[id=187, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
33) Thread[id=207, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
34) Thread[id=208, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-EventThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
35) Thread[id=203, name=qtp4252568-203, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
36) Thread[id=216, name=HashSessionScavenger-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
37) Thread[id=200, name=qtp4252568-200 Acceptor1 SelectChannelConnector@127.0.0.1:39018, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:225)
at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
38) Thread[id=221, name=qtp9125177-221, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
39) Thread[id=227, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
40) Thread[id=183, name=qtp3977146-183, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
41) Thread[id=197, name=qtp4252568-197 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
42) Thread[id=184, name=qtp3977146-184, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:745)
43) Thread[id=211, name=searcherExecutor-293-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
44) Thread[id=191, name=searcherExecutor-284-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
45) Thread[id=241, name=OverseerStateUpdate-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=WAITING, group=Overseer state updater.]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:307)
at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:304)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:74)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:304)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:604)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:209)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([4A09753575938C19]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=179, name=qtp3977146-179 Acceptor0 SelectChannelConnector@127.0.0.1:40891, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=217, name=qtp9125177-217 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:106) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=220, name=qtp9125177-220 Acceptor1 SelectChannelConnector@127.0.0.1:51822, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=218, name=qtp9125177-218 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:106) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=231, name=searcherExecutor-302-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=196, name=HashSessionScavenger-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 7) Thread[id=212, name=Thread-78, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 8) Thread[id=176, name=HashSessionScavenger-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 9) Thread[id=192, name=Thread-67, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 10) Thread[id=178, name=qtp3977146-178 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.isInterrupted(Native Method) at java.lang.Thread.isInterrupted(Thread.java:1000) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=216, name=HashSessionScavenger-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) at java.util.TimerThread.run(Timer.java:505) 12) Thread[id=207, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994) 13) Thread[id=187, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003) 14) Thread[id=227, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994) 15) Thread[id=200, name=qtp4252568-200 Acceptor1 SelectChannelConnector@127.0.0.1:39018, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=197, name=qtp4252568-197 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193) at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210) at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:106) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=211, name=searcherExecutor-293-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=191, name=searcherExecutor-284-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=177, name=qtp3977146-177 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=198, name=qtp4252568-198 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569) at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=199, name=qtp4252568-199 Acceptor0 SelectChannelConnector@127.0.0.1:39018, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=214, name=coreZkRegister-288-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=234, name=coreZkRegister-297-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=232, name=Thread-89, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.solr.core.CloserThread.run(CoreContainer.java:905) 25) Thread[id=194, name=coreZkRegister-279-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=243, name=OverseerHdfsCoreFailoverThread-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:136) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=219, name=qtp9125177-219 Acceptor0 SelectChannelConnector@127.0.0.1:51822, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=180, name=qtp3977146-180 Acceptor1 SelectChannelConnector@127.0.0.1:40891, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest] at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=179, name=qtp3977146-179 Acceptor0 SelectChannelConnector@127.0.0.1:40891, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=217, name=qtp9125177-217 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.IOUtil.drain(Native Method)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:106)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=220, name=qtp9125177-220 Acceptor1 SelectChannelConnector@127.0.0.1:51822, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=218, name=qtp9125177-218 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:193)
at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:106)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=231, name=searcherExecutor-302-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=196, name=HashSessionScavenger-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
7) Thread[id=212, name=Thread-78, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
8) Thread[id=176, name=HashSessionScavenger-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
9) Thread[id=192, name=Thread-67, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:503)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
10) Thread[id=178, name=qtp3977146-178 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.isInterrupted(Native Method)
at java.lang.Thread.isInterrupted(Thread.java:1000)
at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:215)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:78)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
11) Thread[id=216, name=HashSessionScavenger-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
at java.util.TimerThread.run(Timer.java:505)
12) Thread[id=207, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
13) Thread[id=187, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
14) Thread[id=227, name=TEST-LeaderFailoverAfterPartitionTest.testDistribSearch-seed#[4A09753575938C19]-SendThread(127.0.0.1:45632), state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
15) Thread[id=200, name=qtp4252568-200 Acceptor1 SelectChannelConnector@127.0.0.1:39018, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.
[...truncated too long message...]
c.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 19) Thread[id=177, name=qtp3977146-177 Selector0, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
[junit4] > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 20) Thread[id=198, name=qtp4252568-198 Selector1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:90)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
[junit4] > at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 21) Thread[id=199, name=qtp4252568-199 Acceptor0 SelectChannelConnector@127.0.0.1:39018, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:102)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 22) Thread[id=214, name=coreZkRegister-288-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 23) Thread[id=234, name=coreZkRegister-297-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 24) Thread[id=232, name=Thread-89, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:503)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:905)
[junit4] > 25) Thread[id=194, name=coreZkRegister-279-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 26) Thread[id=243, name=OverseerHdfsCoreFailoverThread-92843468793053190-127.0.0.1:41007_ddkj-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:136)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 27) Thread[id=219, name=qtp9125177-219 Acceptor0 SelectChannelConnector@127.0.0.1:51822, state=BLOCKED, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 28) Thread[id=180, name=qtp3977146-180 Acceptor1 SelectChannelConnector@127.0.0.1:40891, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
[junit4] > at org.eclipse.jetty.server.nio.SelectChannelConnector.getConnection(SelectChannelConnector.java:160)
[junit4] > at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:934)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([4A09753575938C19]:0)
[junit4] Completed on J0 in 162.99s, 1 test, 1 failure, 3 errors <<< FAILURES!
[...truncated 1244 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:474: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:454: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/solr/common-build.xml:494: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:1350: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.10-Linux/lucene/common-build.xml:970: There were test failures: 421 suites, 1742 tests, 3 suite-level errors, 1 error, 1308 ignored (14 assumptions)
Total time: 86 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.7.0_67 -client -XX:+UseConcMarkSweepGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any