You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/06/05 12:23:07 UTC

[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 833 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/833/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseG1GC

21 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=5967, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=5970, name=OverseerCollectionConfigSetProcessor-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=5940, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   11) Thread[id=5943, name=OverseerCollectionConfigSetProcessor-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   13) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=5969, name=OverseerStateUpdate-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   16) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   17) Thread[id=5942, name=OverseerStateUpdate-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=5967, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=5970, name=OverseerCollectionConfigSetProcessor-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=5940, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  11) Thread[id=5943, name=OverseerCollectionConfigSetProcessor-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  13) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=5969, name=OverseerStateUpdate-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  16) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  17) Thread[id=5942, name=OverseerStateUpdate-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    3) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    6) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)         at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)         at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)         at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)         at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   11) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=5978, name=zkCallback-1107-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=5939, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:44025), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   3) Thread[id=5945, name=Thread-2171, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   4) Thread[id=5974, name=zkCallback-1107-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=5972, name=Thread-2173, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   6) Thread[id=5944, name=OverseerHdfsCoreFailoverThread-96018484180746243-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Native Method)
        at java.lang.Throwable.fillInStackTrace(java.base@9-ea/Throwable.java:788)
        at java.lang.Throwable.<init>(java.base@9-ea/Throwable.java:271)
        at java.lang.Exception.<init>(java.base@9-ea/Exception.java:66)
        at java.lang.InterruptedException.<init>(java.base@9-ea/InterruptedException.java:67)
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=5977, name=zkCallback-1107-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=5976, name=zkCallback-1107-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=5971, name=OverseerHdfsCoreFailoverThread-96018505175793667-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=5966, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[49FB930F5CA93324]-SendThread(127.0.0.1:35001), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  11) Thread[id=5975, name=zkCallback-1107-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at http://127.0.0.1:45440/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45440/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:44457: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:44457: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:37553/_/n/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /_/n/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37553/_/n/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /_/n/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:E2018E1A8375B50A]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:46543/c/fs/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /c/fs/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46543/c/fs/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /c/fs/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:37982/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:37982/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:F770F5A025D33D11]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:45973

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:45973
	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 47 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Error from server at http://127.0.0.1:44444/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44444/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.clou

[...truncated too long message...]

   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:244)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:86)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.cancelElection(ElectionContext.java:143)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.cancelElection(ElectionContext.java:279)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:455)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(java.base@9-ea/Executors.java:514)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]   2> 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 5486983 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@528c5cb3{HTTP/1.1,[http/1.1]}{127.0.0.1:35106}
   [junit4]   2> 5486983 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1b210065{/lb/im,null,UNAVAILABLE}
   [junit4]   2> 5486984 INFO  (TEST-RecoveryZkTest.test-seed#[49FB930F5CA93324]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44398 44398
   [junit4]   2> 5487091 INFO  (Thread-2150) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:44398 44398
   [junit4]   2> 5487092 WARN  (Thread-2150) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/collections/collection1/state.json
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/96018520568037385-core_node1-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/96018520568037385-127.0.0.1:33867_lb%2Fim-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=49FB930F5CA93324 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr-Cyrl -Dtests.timezone=Africa/Juba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  328s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([49FB930F5CA93324:C1AFACD5F2555EDC]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 5487093 INFO  (SUITE-RecoveryZkTest-seed#[49FB930F5CA93324]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_49FB930F5CA93324-001
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=15993, maxDocsPerChunk=1, blockSize=4), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=15993, blockSize=4)), sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=sr-Cyrl, timezone=Africa/Juba
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=385556872,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [CacheHeaderTest, TestDefaultStatsCache, TestMergePolicyConfig, TestFieldCacheWithThreads, TestFieldResource, TestDynamicFieldCollectionResource, TestCustomDocTransformer, RequestLoggingTest, JSONWriterTest, TestDocSet, SharedFSAutoReplicaFailoverTest, TestRandomDVFaceting, TestClassicSimilarityFactory, XsltUpdateRequestHandlerTest, ConcurrentDeleteAndCreateCollectionTest, UUIDFieldTest, SmileWriterTest, TestBadConfig, TestLuceneMatchVersion, GraphQueryTest, TestConfigSetProperties, TestSmileRequest, TestLazyCores, TestRebalanceLeaders, TestOmitPositions, JavabinLoaderTest, TestHighlightDedupGrouping, TestManagedSchema, HdfsUnloadDistributedZkTest, SystemInfoHandlerTest, TestSizeLimitedDistributedMap, BasicZkTest, OutputWriterTest, CreateCollectionCleanupTest, MultiThreadedOCPTest, DistributedDebugComponentTest, TestReloadDeadlock, TestAnalyzedSuggestions, TestSubQueryTransformerCrossCore, CopyFieldTest, ZkNodePropsTest, FileBasedSpellCheckerTest, TestSolrQueryParserResource, TestExpandComponent, TestLeaderElectionZkExpiry, URLClassifyProcessorTest, TestUseDocValuesAsStored, TestManagedResourceStorage, TlogReplayBufferedWhileIndexingTest, QueryEqualityTest, WrapperMergePolicyFactoryTest, TestCustomSort, TestConfig, TestSolrConfigHandler, DeleteStatusTest, DisMaxRequestHandlerTest, BasicDistributedZkTest, TestSQLHandler, SolrRequestParserTest, TestManagedSynonymFilterFactory, ForceLeaderTest, DistanceUnitsTest, SSLMigrationTest, DistributedSuggestComponentTest, CdcrReplicationDistributedZkTest, ConjunctionSolrSpellCheckerTest, DeleteLastCustomShardedReplicaTest, TestBinaryField, TestOverriddenPrefixQueryForCustomFieldType, AlternateDirectoryTest, AnalysisErrorHandlingTest, PolyFieldTest, TestLRUCache, TestValueSourceCache, CoreAdminCreateDiscoverTest, TestXmlQParser, OverseerCollectionConfigSetProcessorTest, PrimUtilsTest, TestIBSimilarityFactory, HdfsDirectoryTest, PathHierarchyTokenizerFactoryTest, PreAnalyzedFieldTest, SuggestComponentContextFilterQueryTest, TestRTGBase, LeaderInitiatedRecoveryOnCommitTest, BlockJoinFacetDistribTest, TestDistribDocBasedVersion, OverseerTaskQueueTest, HdfsSyncSliceTest, InfoHandlerTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, HdfsRecoverLeaseTest, TestGraphTermsQParserPlugin, PingRequestHandlerTest, TestRuleBasedAuthorizationPlugin, DistribJoinFromCollectionTest, TestFaceting, BigEndianAscendingWordSerializerTest, TestCrossCoreJoin, VersionInfoTest, TestRemoteStreaming, TestFieldCache, ChaosMonkeySafeLeaderTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest]
   [junit4] Completed [580/613 (18!)] on J0 in 327.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 161 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 15 errors, 4 failures, 1297 ignored (66 assumptions) [seed: 49FB930F5CA93324]

Total time: 160 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 840 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/840/
Java: 64bit/jdk1.8.0_92 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery:     1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestCoreDiscovery

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)




Build Log:
[...truncated 11968 lines...]
   [junit4] Suite: org.apache.solr.core.TestCoreDiscovery
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/init-core-data-001
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1244388 INFO  (SUITE-TestCoreDiscovery-seed#[AE7EF232A10F4145]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1244389 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testAlternateCoreDir
   [junit4]   2> 1244392 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001'
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244393 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/solr.xml
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer New CoreContainer 1385185418
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001]
   [junit4]   2> 1244396 WARN  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001/lib
   [junit4]   2> 1244396 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244398 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244399 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244400 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244401 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002
   [junit4]   2> 1244401 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core2, config=solrconfig-minimal.xml, loadOnStartup=false, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=core2}
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found core core2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=core1}
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1
   [junit4]   2> 1244402 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CorePropertiesLocator Found 2 core definitions
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1'
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244402 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244408 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244410 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244426 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244429 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.s.IndexSchema [core1] Schema name=tiny
   [junit4]   2> 1244432 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244436 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1'
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'core1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore [[core1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/]
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore JMX monitoring not detected for core: core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1
   [junit4]   2> 1244437 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index/
   [junit4]   2> 1244438 WARN  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore [core1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index' doesn't exist. Creating new index...
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@381be64c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27e00987),segFN=segments_1,generation=1}
   [junit4]   2> 1244438 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244439 INFO  (OldIndexDirectoryCleanupThreadForCore-core1) [    x:core1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/
   [junit4]   2> 1244439 WARN  (OldIndexDirectoryCleanupThreadForCore-core1) [    x:core1] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244441 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244443 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244443 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244444 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244444 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244445 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244445 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@381be64c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27e00987),segFN=segments_1,generation=1}
   [junit4]   2> 1244447 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.s.SolrIndexSearcher Opening [Searcher@42a4a269[core1] main]
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf}
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/conf
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244448 INFO  (searcherExecutor-4442-thread-1-processing-x:core1) [    x:core1] o.a.s.c.SolrCore [core1] Registered new searcher Searcher@42a4a269[core1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244448 INFO  (coreLoadExecutor-4441-thread-1) [    x:core1] o.a.s.c.CoreContainer registering core: core1
   [junit4]   2> 1244448 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2'
   [junit4]   2> 1244449 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244449 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-001
   [junit4]   2> 1244454 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244457 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244462 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244466 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.s.IndexSchema [core2] Schema name=tiny
   [junit4]   2> 1244468 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2'
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    ] o.a.s.c.CoreContainer Creating SolrCore 'core2' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore [[core2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/]
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore JMX monitoring not detected for core: core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2
   [junit4]   2> 1244469 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index/
   [junit4]   2> 1244470 WARN  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore [core2] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index' doesn't exist. Creating new index...
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@2ce41f44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36bd269c),segFN=segments_1,generation=1}
   [junit4]   2> 1244470 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244470 INFO  (OldIndexDirectoryCleanupThreadForCore-core2) [    x:core2] o.a.s.c.SolrCore Looking for old index directories to cleanup for core core2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/
   [junit4]   2> 1244471 WARN  (OldIndexDirectoryCleanupThreadForCore-core2) [    x:core2] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244472 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244473 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244474 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244475 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=77.130859375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=29.512037338187863, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@2ce41f44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36bd269c),segFN=segments_1,generation=1}
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244476 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.s.SolrIndexSearcher Opening [Searcher@37c84115[core2] main]
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf}
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/conf
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244477 INFO  (searcherExecutor-4446-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Registered new searcher Searcher@37c84115[core2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244477 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer registering core: core2
   [junit4]   2> 1244478 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1385185418
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2]  CLOSING SolrCore org.apache.solr.core.SolrCore@33e9c1e2
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.SolrCoreState Closing SolrCoreState
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1244478 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrCore [core2] Closing main searcher on request.
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2 [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2;done=false>>]
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index;done=false>>]
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244479 INFO  (coreCloseExecutor-4450-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core2/core2/index
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.SolrCore [core1]  CLOSING SolrCore org.apache.solr.core.SolrCore@440e8e7f
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.SolrCoreState Closing SolrCoreState
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1244494 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.SolrCore [core1] Closing main searcher on request.
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1 [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1;done=false>>]
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory looking to close /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index [CachedDir<<refCount=0;path=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index;done=false>>]
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 1244495 INFO  (coreCloseExecutor-4450-thread-1-processing-x:core2) [    x:core1] o.a.s.c.CachingDirectoryFactory Closing directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-002/core1/core1/index
   [junit4]   2> 1244495 INFO  (TEST-TestCoreDiscovery.testAlternateCoreDir-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Ending testAlternateCoreDir
   [junit4]   2> 1244497 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Starting testDuplicateNames
   [junit4]   2> 1244500 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003'
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244501 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/solr.xml
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer New CoreContainer 511314936
   [junit4]   2> 1244503 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003]
   [junit4]   2> 1244504 WARN  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/lib
   [junit4]   2> 1244504 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244504 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003
   [junit4]   2> 1244505 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=false, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=stuffandnonsense}
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/core2
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=core1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=stuffandnonsense}
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core core1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-003/core1
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found 2 core definitions
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Shutting down CoreContainer instance=511314936
   [junit4]   2> 1244506 INFO  (TEST-TestCoreDiscovery.testDuplicateNames-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Ending testDuplicateNames
   [junit4]   2> 1244508 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.SolrTestCaseJ4 ###Starting testTooManyTransientCores
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004'
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244516 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/solr.xml
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer New CoreContainer 243489109
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004]
   [junit4]   2> 1244519 WARN  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Couldn't add files from /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/lib to classpath: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/lib
   [junit4]   2> 1244519 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1244520 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.SolrCores Allocating transient cache for 2 transient cores
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1244521 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT5, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT5}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT5 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT1, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT1}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreLOS, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=false, dataDir=coreLOS}
   [junit4]   2> 1244522 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreLOS in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT4, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT4}
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT4 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT4
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT2, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT2}
   [junit4]   2> 1244523 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT2 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT2
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT6, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT6}
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT6 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT6
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=coreT3, config=solrconfig-minimal.xml, loadOnStartup=true, schema=schema-tiny.xml, configSetProperties=configsetprops.json, transient=true, dataDir=coreT3}
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found core coreT3 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT3
   [junit4]   2> 1244524 INFO  (TEST-TestCoreDiscovery.testTooManyTransientCores-seed#[AE7EF232A10F4145]) [    x:core2] o.a.s.c.CorePropertiesLocator Found 7 core definitions
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5'
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244525 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1'
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS'
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244529 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244533 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244533 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244534 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1244534 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244549 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244552 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244554 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreT1] Schema name=tiny
   [junit4]   2> 1244556 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1'
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreT1' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1
   [junit4]   2> 1244557 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore [[coreT1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/]
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore JMX monitoring not detected for core: coreT1
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1
   [junit4]   2> 1244558 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
   [junit4]   2> 1244562 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244563 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig-minimal.xml
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index/
   [junit4]   2> 1244564 WARN  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore [coreT1] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index' doesn't exist. Creating new index...
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/index
   [junit4]   2> 1244564 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@532e2ba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26edd96f),segFN=segments_1,generation=1}
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreT5] Schema name=tiny
   [junit4]   2> 1244565 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema [coreLOS] Schema name=tiny
   [junit4]   2> 1244567 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244567 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1244567 INFO  (OldIndexDirectoryCleanupThreadForCore-coreT1) [    x:coreT1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreT1 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/
   [junit4]   2> 1244567 WARN  (OldIndexDirectoryCleanupThreadForCore-coreT1) [    x:coreT1] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/coreT1/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5'
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreT5' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS'
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:core2] o.a.s.c.CoreContainer Creating SolrCore 'coreLOS' using configuration from instancedir /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore [[coreT5] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/]
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore [[coreLOS] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/]
   [junit4]   2> 1244568 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore JMX monitoring not detected for core: coreT5
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore JMX monitoring not detected for core: coreLOS
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index/
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore New index directory detected: old=null new=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index/
   [junit4]   2> 1244569 WARN  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore [coreT5] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index' doesn't exist. Creating new index...
   [junit4]   2> 1244569 WARN  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore [coreLOS] Solr index directory '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index' doesn't exist. Creating new index...
   [junit4]   2> 1244569 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/index
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CachingDirectoryFactory return new directory for /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/index
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2763671875, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=1.3293771542077448, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@6071c9f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e2fcf84),segFN=segments_1,generation=1}
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@7e01285f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e5eced1),segFN=segments_1,generation=1}
   [junit4]   2> 1244570 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244570 INFO  (OldIndexDirectoryCleanupThreadForCore-coreT5) [    x:coreT5] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreT5 in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/
   [junit4]   2> 1244570 WARN  (OldIndexDirectoryCleanupThreadForCore-coreT5) [    x:coreT5] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/coreT5/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244571 INFO  (OldIndexDirectoryCleanupThreadForCore-coreLOS) [    x:coreLOS] o.a.s.c.SolrCore Looking for old index directories to cleanup for core coreLOS in /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/
   [junit4]   2> 1244571 WARN  (OldIndexDirectoryCleanupThreadForCore-coreLOS) [    x:coreLOS] o.a.s.c.DirectoryFactory /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/coreLOS/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244572 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244573 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244574 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244575 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@6071c9f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e2fcf84),segFN=segments_1,generation=1}
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244576 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.s.SolrIndexSearcher Opening [Searcher@6a3f5db8[coreT5] main]
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT5/conf
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (searcherExecutor-4466-thread-1-processing-x:coreT5) [    x:coreT5] o.a.s.c.SolrCore [coreT5] Registered new searcher Searcher@6a3f5db8[coreT5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.SolrCores Opening transient core coreT5
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:coreT5] o.a.s.c.CoreContainer registering core: coreT5
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@532e2ba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26edd96f),segFN=segments_1,generation=1}
   [junit4]   2> 1244577 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT4'
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6196029480165364]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@7e01285f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e5eced1),segFN=segments_1,generation=1}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.s.SolrIndexSearcher Opening [Searcher@154d1d1c[coreT1] main]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-1-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader using system property solr.solr.home: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.s.SolrIndexSearcher Opening [Searcher@58a9e044[coreLOS] main]
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreT1/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf}
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.core.TestCoreDiscovery_AE7EF232A10F4145-001/tempDir-004/coreLOS/conf
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244578 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1244579 INFO  (searcherExecutor-4465-thread-1-processing-x:coreT1) [    x:coreT1] o.a.s.c.SolrCore [coreT1] Registered new searcher Searcher@154d1d1c[coreT1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.SolrCores Opening transient core coreT1
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:coreT1] o.a.s.c.CoreContainer registering core: coreT1
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-3-processing-x:core2) [    x:coreLOS] o.a.s.c.CoreContainer registering core: coreLOS
   [junit4]   2> 1244579 INFO  (coreLoadExecutor-4464-thread-2-processing-x:core2) [    x:core2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/home/jenkins/workspa

[...truncated too long message...]

Thread.java:745)
   [junit4]   2> 
   [junit4]   2> Jun 07, 2016 2:17:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Jun 07, 2016 2:17:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Jun 07, 2016 2:17:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2> Jun 07, 2016 2:17:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Lucene60, sim=ClassicSimilarity, locale=et, timezone=America/Regina
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=2,free=116467560,total=518148096
   [junit4]   2> NOTE: All tests run in this JVM: [NoCacheHeaderTest, TestSolrQueryParser, TestCryptoKeys, PreAnalyzedUpdateProcessorTest, TestPerFieldSimilarity, ZkStateWriterTest, CleanupOldIndexTest, TestConfigSetImmutable, MergeStrategyTest, TestExactSharedStatsCache, TestInitParams, TestSolrIndexConfig, BJQParserTest, HdfsNNFailoverTest, TestFuzzyAnalyzedSuggestions, StatelessScriptUpdateProcessorFactoryTest, HighlighterConfigTest, HardAutoCommitTest, ShowFileRequestHandlerTest, TestSolrXml, SuggesterTest, TestReload, TestScoreJoinQPScore, TestRestoreCore, TestDistributedMissingSort, ResponseHeaderTest, TestShardHandlerFactory, TestFieldTypeResource, SegmentsInfoRequestHandlerTest, CdcrRequestHandlerTest, TestPKIAuthenticationPlugin, TestSolrDeletionPolicy1, TermVectorComponentDistributedTest, DirectSolrConnectionTest, BasicAuthIntegrationTest, PrimitiveFieldTypeTest, BlockJoinFacetSimpleTest, BigEndianAscendingWordDeserializerTest, LukeRequestHandlerTest, TestCollationField, TestFastWriter, ShardSplitTest, LeaderElectionTest, TestCloudInspectUtil, TestSchemaSimilarityResource, AtomicUpdatesTest, TestLockTree, TestRandomCollapseQParserPlugin, TestStressCloudBlindAtomicUpdates, XmlUpdateRequestHandlerTest, TestSweetSpotSimilarityFactory, SpatialHeatmapFacetsTest, OverseerStatusTest, TestRandomRequestDistribution, TestTolerantSearch, TestRecovery, DistributedFacetPivotLargeTest, TestSolrQueryParserDefaultOperatorResource, TestZkChroot, TestSort, StatsComponentTest, HdfsChaosMonkeyNothingIsSafeTest, TestConfigSetsAPI, AssignTest, BadComponentTest, TestIndexingPerformance, DistributedFacetPivotSmallTest, RemoteQueryErrorTest, TestPostingsSolrHighlighter, TestReversedWildcardFilterFactory, TestHdfsUpdateLog, TestIndexSearcher, TestRTimerTree, TestWriterPerf, RegexBoostProcessorTest, MigrateRouteKeyTest, QueryParsingTest, TestComplexPhraseQParserPlugin, TestJmxMonitoredMap, RecoveryZkTest, TestRequestForwarding, UUIDUpdateProcessorFallbackTest, PluginInfoTest, SimpleCollectionCreateDeleteTest, HLLUtilTest, TestSolrQueryResponse, HdfsTlogReplayBufferedWhileIndexingTest, TestNumericTerms32, TestGroupingSearch, TestBulkSchemaAPI, EnumFieldTest, DistributedVersionInfoTest, TestTolerantUpdateProcessorRandomCloud, DistribDocExpirationUpdateProcessorTest, DebugComponentTest, RequestHandlersTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestDistributedSearch, DistributedSpellCheckComponentTest, TestStressVersions, SimpleFacetsTest, TestSolr4Spatial, SolrCmdDistributorTest, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, TestBadConfig, SoftAutoCommitTest, SimplePostToolTest, TestCoreDiscovery]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCoreDiscovery -Dtests.seed=AE7EF232A10F4145 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=et -Dtests.timezone=America/Regina -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestCoreDiscovery (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MockDirectoryWrapper]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestCoreDiscovery: 
   [junit4]    >    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=8920, name=searcherExecutor-4505-thread-1, state=WAITING, group=TGRP-TestCoreDiscovery]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE7EF232A10F4145]:0)
   [junit4] Completed [446/613 (1!)] on J2 in 145.29s, 12 tests, 1 failure, 2 errors <<< FAILURES!

[...truncated 551 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 3 suite-level errors, 682 ignored (67 assumptions) [seed: AE7EF232A10F4145]

Total time: 48 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 839 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/839/
Java: 64bit/jdk-9-ea+121 -XX:+UseCompressedOops -XX:+UseParallelGC

17 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at http://127.0.0.1:45528/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45528/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.SimpleCollectionCreateDeleteTest.test

Error Message:
No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    2) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=1082, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    5) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=1057, name=OverseerStateUpdate-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=1085, name=OverseerCollectionConfigSetProcessor-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=1058, name=OverseerCollectionConfigSetProcessor-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   12) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=1055, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   14) Thread[id=1084, name=OverseerStateUpdate-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   16) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   17) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   2) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=1082, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   5) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=1057, name=OverseerStateUpdate-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=1085, name=OverseerCollectionConfigSetProcessor-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=1058, name=OverseerCollectionConfigSetProcessor-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  12) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  13) Thread[id=1055, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  14) Thread[id=1084, name=OverseerStateUpdate-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  16) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  17) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    2) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    6) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1081, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:42177), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   2) Thread[id=1087, name=Thread-391, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=1093, name=zkCallback-183-thread-6, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=1060, name=Thread-389, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=1054, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[B721922F7358ADDD]-SendThread(127.0.0.1:34628), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   6) Thread[id=1086, name=OverseerHdfsCoreFailoverThread-96026543810281475-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=1059, name=OverseerHdfsCoreFailoverThread-96026522827489283-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=1091, name=zkCallback-183-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=1089, name=zkCallback-183-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=1090, name=zkCallback-183-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=1092, name=zkCallback-183-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at https://127.0.0.1:42951/lee/v/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /lee/v/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42951/lee/v/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /lee/v/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:33488/at_jgc/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /at_jgc/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33488/at_jgc/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /at_jgc/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:34084: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:34084: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:34798/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolr

[...truncated too long message...]

rDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B721922F7358ADDD:3F75ADF5DDA4C025]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10414172 INFO  (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[B721922F7358ADDD]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_B721922F7358ADDD-001
   [junit4]   2> Jun 06, 2016 11:54:29 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=242, maxMBSortInHeap=5.517363650119625, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=lu, timezone=Antarctica/Casey
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=160036344,total=455606272
   [junit4]   2> NOTE: All tests run in this JVM: [TestQuerySenderListener, TestStressUserVersions, ChaosMonkeySafeLeaderTest, DistanceFunctionTest, TestScoreJoinQPNoScore, TestConfigSets, TestLegacyFieldCache, FileUtilsTest, TestQueryUtils, TestPivotHelperCode, TestSolr4Spatial2, TestUniqueKeyFieldResource, CollectionsAPIDistributedZkTest, TestChildDocTransformer, TestMiniSolrCloudClusterBase, SpellPossibilityIteratorTest, DirectSolrSpellCheckerTest, DeleteReplicaTest, HdfsWriteToMultipleCollectionsTest, ExitableDirectoryReaderTest, TestFieldTypeCollectionResource, DistributedQueueTest, TestTrie, OpenCloseCoreStressTest, SolrCmdDistributorTest, HighlighterMaxOffsetTest, TestSha256AuthenticationProvider, SharedFSAutoReplicaFailoverUtilsTest, TestLRUStatsCache, TestGeoJSONResponseWriter, HdfsRecoveryZkTest, UniqFieldsUpdateProcessorFactoryTest, HdfsBasicDistributedZkTest, TestAnalyzeInfixSuggestions, TestDistribIDF, AsyncMigrateRouteKeyTest, TestPartialUpdateDeduplication, TestSolrCoreProperties, MultiTermTest, TestFieldCacheReopen, TestDistributedGrouping, OutOfBoxZkACLAndCredentialsProvidersTest, TestCoreContainer, ShardRoutingCustomTest, CSVRequestHandlerTest, TestStressVersions, TestTestInjection, TestUseDocValuesAsStored2, SpellCheckComponentTest, AliasIntegrationTest, TestSubQueryTransformerDistrib, TestFiltering, TolerantUpdateProcessorTest, ZkStateReaderTest, DistributedIntervalFacetingTest, RestartWhileUpdatingTest, SolrXmlInZkTest, NotRequiredUniqueKeyTest, TestFreeTextSuggestions, TestTolerantUpdateProcessorCloud, SuggesterFSTTest, TestCollectionAPI, DistributedExpandComponentTest, SecurityConfHandlerTest, HttpPartitionTest, HLLSerializationTest, TestBlobHandler, TestConfigOverlay, CollectionReloadTest, SolrInfoMBeanTest, TestSurroundQueryParser, DistributedMLTComponentTest, TestManagedSchemaAPI, TestDynamicFieldResource, BlockDirectoryTest, DistributedQueryComponentCustomSortTest, TestFieldSortValues, RecoveryAfterSoftCommitTest, TestDownShardTolerantSearch, ShardRoutingTest, TestSolrDeletionPolicy2, TestCursorMarkWithoutUniqueKey, BadCopyFieldTest, DOMUtilTest, BasicFunctionalityTest, TestRestManager, TestCloudDeleteByQuery, TestReplicaProperties, TestSolrConfigHandlerCloud, TestInfoStreamLogging, HdfsCollectionsAPIDistributedZkTest, TestQuerySenderNoQuery, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed [607/613 (14!)] on J1 in 612.35s, 1 test, 1 error <<< FAILURES!

[...truncated 38 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 11 errors, 4 failures, 1777 ignored (63 assumptions) [seed: B721922F7358ADDD]

Total time: 197 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 838 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/838/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseG1GC

22 tests failed.
FAILED:  org.apache.solr.schema.TestCloudManagedSchema.test

Error Message:
Schema resource name differs from expected name expected:<[managed-schema]> but was:<[schema.xml]>

Stack Trace:
org.junit.ComparisonFailure: Schema resource name differs from expected name expected:<[managed-schema]> but was:<[schema.xml]>
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.apache.solr.schema.TestCloudManagedSchema.test(TestCloudManagedSchema.java:68)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.CollectionTooManyReplicasTest.testAddTooManyReplicas

Error Message:
No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 10000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:54D2E68CA341AE7F]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:427)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:990)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
15 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=7095, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7072, name=OverseerCollectionConfigSetProcessor-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=7098, name=OverseerCollectionConfigSetProcessor-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7069, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   10) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=7097, name=OverseerStateUpdate-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   13) Thread[id=7071, name=OverseerStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=7095, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7072, name=OverseerCollectionConfigSetProcessor-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=7098, name=OverseerCollectionConfigSetProcessor-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7069, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  10) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=7097, name=OverseerStateUpdate-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  13) Thread[id=7071, name=OverseerStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:35328/n_/p: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:35328/n_/p: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:44310/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44310/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:727A8F41327B17E0]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:45997/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:45997/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:670BF4FB94DD9FFB]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:46176

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:46176
	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE:51D4AD8E435BFC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(java.base@9-ea/Native Method)
	at java.net.SocketInputStream.socketRead(java.base@9-ea/SocketInputStream.java:116)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:170)
	at java.net.SocketInputStream.read(java.base@9-ea/SocketInputStream.java:141)
	at sun.security.ssl.SSLSocketInputRecord.read(java.base@9-ea/SSLSocketInputRecord.java:425)
	at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(java.base@9-ea/SSLSocketInputRecord.java:65)
	at sun.security.ssl.SSLSocketImpl.bytesInCompletePacket(java.base@9-ea/SSLSocketImpl.java:911)
	at sun.security.ssl.AppInputStream.read(java.base@9-ea/AppInputStream.java:144)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 4

[...truncated too long message...]

rStateUpdate-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
   [junit4]    >         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
   [junit4]    >         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >   14) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >   15) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=7102, name=zkCallback-1274-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    2) Thread[id=7074, name=Thread-2371, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Native Method)
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Object.java:516)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   [junit4]    >    3) Thread[id=7094, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:39710), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    >    4) Thread[id=7103, name=zkCallback-1274-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    5) Thread[id=7099, name=OverseerHdfsCoreFailoverThread-96025796035018755-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    6) Thread[id=7104, name=zkCallback-1274-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    7) Thread[id=7100, name=Thread-2373, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Native Method)
   [junit4]    >         at java.lang.Object.wait(java.base@9-ea/Object.java:516)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   [junit4]    >    8) Thread[id=7073, name=OverseerHdfsCoreFailoverThread-96025775042068483-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
   [junit4]    >         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    >    9) Thread[id=7068, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[D9809254EDA791CE]-SendThread(127.0.0.1:35172), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
   [junit4]    >         at java.lang.Thread.sleep(java.base@9-ea/Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D9809254EDA791CE]:0)
   [junit4] Completed [317/613 (19!)] on J1 in 665.76s, 2 tests, 2 failures, 2 errors <<< FAILURES!

[...truncated 999 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 15 errors, 5 failures, 1110 ignored (72 assumptions) [seed: D9809254EDA791CE]

Total time: 183 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 837 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/837/
Java: 64bit/jdk-9-ea+121 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

20 tests failed.
FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
[[], [], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], []]

Stack Trace:
java.lang.AssertionError: [[], [], [], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"]], []]
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.schema.TestBulkSchemaConcurrent.test(TestBulkSchemaConcurrent.java:115)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    3) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=7016, name=OverseerStateUpdate-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=7042, name=OverseerStateUpdate-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=7017, name=OverseerCollectionConfigSetProcessor-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   12) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   13) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=7043, name=OverseerCollectionConfigSetProcessor-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=7040, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   16) Thread[id=7014, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   3) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=7016, name=OverseerStateUpdate-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=7042, name=OverseerStateUpdate-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=7017, name=OverseerCollectionConfigSetProcessor-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  12) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  13) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=7043, name=OverseerCollectionConfigSetProcessor-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  15) Thread[id=7040, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  16) Thread[id=7014, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    2) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    4) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    6) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    7) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    8) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   10) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7047, name=zkCallback-1123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   2) Thread[id=7050, name=zkCallback-1123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=7044, name=OverseerHdfsCoreFailoverThread-96024591208284163-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   4) Thread[id=7018, name=OverseerHdfsCoreFailoverThread-96024570223591427-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=7013, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:43486), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   6) Thread[id=7019, name=Thread-2827, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   7) Thread[id=7039, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[13E501B446D4C3CA]-SendThread(127.0.0.1:36307), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   8) Thread[id=7048, name=zkCallback-1123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=7045, name=Thread-2829, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  10) Thread[id=7049, name=zkCallback-1123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA]:0)


FAILED:  org.apache.solr.cloud.AliasIntegrationTest.test

Error Message:
Error from server at https://127.0.0.1:40154/_dwu/ey/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_dwu/ey/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40154/_dwu/ey/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_dwu/ey/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.AliasIntegrationTest.test(AliasIntegrationTest.java:69)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:38584/qgkz/o/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /qgkz/o/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:38584/qgkz/o/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /qgkz/o/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:38695: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:38695: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:33368/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33368/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicZkTest.testBasic

Error Message:
No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 30000ms , collection: collection1 slice: shard1
	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:B81F1CA1990845E4]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:718)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:685)
	at org.apache.solr.cloud.BasicZkTest.testBasic(BasicZkTest.java:51)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.te

[...truncated too long message...]

e.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([13E501B446D4C3CA:9BB13E6EE828AE32]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
   [junit4]    > 	at org.apache.solr.cloud.RemoteQueryErrorTest.test(RemoteQueryErrorTest.java:49)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 8677219 INFO  (SUITE-RemoteQueryErrorTest-seed#[13E501B446D4C3CA]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.RemoteQueryErrorTest_13E501B446D4C3CA-001
   [junit4]   2> Jun 06, 2016 1:38:01 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1897, maxMBSortInHeap=7.52727781057156, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=de-AT, timezone=Pacific/Tongatapu
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=81611880,total=508887040
   [junit4]   2> NOTE: All tests run in this JVM: [RulesTest, DocValuesMultiTest, DeleteShardTest, TestDistributedMissingSort, TestSolrIndexConfig, HdfsTlogReplayBufferedWhileIndexingTest, DirectUpdateHandlerOptimizeTest, AnalyticsMergeStrategyTest, TestElisionMultitermQuery, SolrTestCaseJ4Test, TestGroupingSearch, TestExactSharedStatsCache, MergeStrategyTest, CleanupOldIndexTest, SimpleFacetsTest, DistribDocExpirationUpdateProcessorTest, DistributedFacetPivotSmallTest, DocExpirationUpdateProcessorFactoryTest, SolrCoreCheckLockOnStartupTest, CollectionStateFormat2Test, XmlUpdateRequestHandlerTest, TermVectorComponentDistributedTest, TestFieldTypeResource, OverseerTest, FullSolrCloudDistribCmdsTest, DirectUpdateHandlerTest, TestHdfsUpdateLog, TestCloudManagedSchema, BigEndianAscendingWordDeserializerTest, TestSolrXml, DirectSolrConnectionTest, TestRestoreCore, TestPKIAuthenticationPlugin, BlockJoinFacetSimpleTest, TestJettySolrRunner, BadIndexSchemaTest, RegexBoostProcessorTest, PreAnalyzedUpdateProcessorTest, UpdateParamsTest, TestPerFieldSimilarity, DistributedQueryComponentOptimizationTest, TestConfigReload, TestSolrFieldCacheMBean, TestFilteredDocIdSet, CurrencyFieldXmlFileTest, TestSolrConfigHandlerConcurrent, TestNumericTerms32, TestShardHandlerFactory, TestCSVResponseWriter, TestCoreDiscovery, StatsComponentTest, TestTolerantUpdateProcessorRandomCloud, HardAutoCommitTest, TestDynamicLoading, ZkStateWriterTest, RollingRestartTest, TestReversedWildcardFilterFactory, TestPHPSerializedResponseWriter, DataDrivenBlockJoinTest, PluginInfoTest, TestTolerantSearch, DateMathParserTest, TestExclusionRuleCollectionAccess, TestLockTree, RemoteQueryErrorTest]
   [junit4] Completed [578/613 (17!)] on J2 in 615.95s, 1 test, 1 error <<< FAILURES!

[...truncated 152 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 14 errors, 4 failures, 1150 ignored (68 assumptions) [seed: 13E501B446D4C3CA]

Total time: 181 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Linux (64bit/jdk1.8.0_92) - Build # 836 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/836/
Java: 64bit/jdk1.8.0_92 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

12 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Doc with id=2 not found in http://127.0.0.1:35483/qkaxn/c8n_1x2_leader_session_loss due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=2 not found in http://127.0.0.1:35483/qkaxn/c8n_1x2_leader_session_loss due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:604)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:559)
	at org.apache.solr.cloud.HttpPartitionTest.testLeaderZkSessionLoss(HttpPartitionTest.java:507)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:120)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=1092, name=OverseerCollectionConfigSetProcessor-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    5) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=1062, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    7) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    8) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)   10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=1089, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   12) Thread[id=1064, name=OverseerStateUpdate-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)   14) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=1065, name=OverseerCollectionConfigSetProcessor-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)         at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)         at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=1091, name=OverseerStateUpdate-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=1092, name=OverseerCollectionConfigSetProcessor-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   5) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=1062, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   7) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   8) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
  10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=1089, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  12) Thread[id=1064, name=OverseerStateUpdate-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer state updater.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
  14) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=1065, name=OverseerCollectionConfigSetProcessor-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=WAITING, group=Overseer collection creation process.]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
        at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=1091, name=OverseerStateUpdate-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    9) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1098, name=zkCallback-211-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=1093, name=OverseerHdfsCoreFailoverThread-96022541866565635-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1094, name=Thread-381, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   4) Thread[id=1066, name=OverseerHdfsCoreFailoverThread-96022520875778051-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=1088, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:46786), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   6) Thread[id=1097, name=zkCallback-211-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=1067, name=Thread-379, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=1061, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[7FC47D48B54585CC]-SendThread(127.0.0.1:44336), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   9) Thread[id=1096, name=zkCallback-211-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=1099, name=zkCallback-211-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC]:0)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:39972: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:39972: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Error from server at http://127.0.0.1:36216/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36216/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:128)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration

Error Message:
Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore

Stack Trace:
org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:C14F1BE7CC3F8BF9]:0)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
	... 41 more
Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	... 42 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:42963

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:42963
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoConfigSetExist(CollectionsAPIDistributedZkTest.java:601)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:176)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
	... 47 more


FAILED:  org.apache.solr.cloud.DistribCursorPagingTest.test

Error Message:
Error from server at http://127.0.0.1:35284/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:35284/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.DistribCursorPagingTest.doBadInputTest(DistribCursorPagingTest.java:114)
	at org.apache.solr.cloud.DistribCursorPagingTest.test(DistribCursorPagingTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:D6F3D8EB92600D27]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderElectionIntegrationTest.testSimpleSliceLeaderElection

Error Message:
Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1

Stack Trace:
java.lang.AssertionError: Could not start container:7000. Reason:class org.apache.solr.common.SolrException Error CREATEing SolrCore 'collection1_shard1': Unable to create core [collection1_shard1] Caused by: Timed out getting coreNodeName for collection1_shard1
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:21F0B006F63A192B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderElectionIntegrationTest.setUp(LeaderElectionIntegrationTest.java:102)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ShardRoutingTest.test

Error Message:
Error from server at https://127.0.0.1:35969/rw_zzh/co/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 503 </title> </head> <body> <h2>HTTP ERROR: 503</h2> <p>Problem accessing /rw_zzh/co/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35969/rw_zzh/co/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 503 </title>
</head>
<body>
<h2>HTTP ERROR: 503</h2>
<p>Problem accessing /rw_zzh/co/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore is loading,code=503}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:F79042921BB9E834]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
	at org.apache.solr.BaseDistributedSearchTestCase.index(BaseDistributedSearchTestCase.java:483)
	at org.apache.solr.cloud.ShardRoutingTest.doAddDoc(ShardRoutingTest.java:324)
	at org.apache.solr.cloud.ShardRoutingTest.doHashingTest(ShardRoutingTest.java:132)
	at org.apache.solr.cloud.ShardRoutingTest.test(ShardRoutingTest.java:109)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10691 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_7FC47D48B54585CC-001/init-core-data-001
   [junit4]   2> 0    INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.e.j.u.log Logging initialized @1720ms
   [junit4]   2> 6    INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 14   INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[7FC47D48B54585CC]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 38   INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 40   INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 40   INFO  (Thread-1) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 140  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.ZkTestServer start zk server on port:35617
   [junit4]   2> 161  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 192  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 245  INFO  (zkCallback-1-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@77e66d38 name:ZooKeeperConnection Watcher:127.0.0.1:35617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 245  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 247  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 249  INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[7FC47D48B54585CC]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 277  WARN  (NIOServerCxn.Factory

[...truncated too long message...]

127.0.0.1:1663_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 5290364 ERROR (OverseerHdfsCoreFailoverThread-96022820617125895-127.0.0.1:1663_solr-n_0000000002) [n:127.0.0.1:1663_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread OverseerAutoReplicaFailoverThread had an error in its thread work loop.:org.apache.solr.common.SolrException: Could not load collection from ZK: collection1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1050)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:610)
   [junit4]   2> 	at org.apache.solr.common.cloud.ClusterState.getCollectionsMap(ClusterState.java:248)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.doWork(OverseerAutoReplicaFailoverThread.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:129)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1342)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1059)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1045)
   [junit4]   2> 	... 5 more
   [junit4]   2> 
   [junit4]   2> 5290364 WARN  (zkCallback-1020-thread-2-processing-n:127.0.0.1:1663_solr) [n:127.0.0.1:1663_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 5290366 INFO  (TEST-ClusterStateUpdateTest.testCoreRegistration-seed#[7FC47D48B54585CC]) [n:127.0.0.1:1661_solr    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38874 38874
   [junit4]   2> 5290384 INFO  (Thread-2164) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38874 38874
   [junit4]   2> 5290385 WARN  (Thread-2164) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/security.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 
   [junit4]   2> 5290385 INFO  (TEST-ClusterStateUpdateTest.testCoreRegistration-seed#[7FC47D48B54585CC]) [n:127.0.0.1:1661_solr    ] o.a.s.SolrTestCaseJ4 ###Ending testCoreRegistration
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ClusterStateUpdateTest -Dtests.method=testCoreRegistration -Dtests.seed=7FC47D48B54585CC -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=hi -Dtests.timezone=Asia/Hebron -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    322s J1 | ClusterStateUpdateTest.testCoreRegistration <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: Error CREATEing SolrCore 'testcore': Unable to create core [testcore] Caused by: Timed out getting shard id for core: testcore
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FC47D48B54585CC:C14F1BE7CC3F8BF9]:0)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:785)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:718)
   [junit4]    > 	at org.apache.solr.cloud.ClusterStateUpdateTest.testCoreRegistration(ClusterStateUpdateTest.java:158)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Unable to create core [testcore]
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:828)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:751)
   [junit4]    > 	... 41 more
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting shard id for core: testcore
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForShardId(ZkController.java:1407)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1362)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	... 42 more
   [junit4]   2> 5290386 INFO  (SUITE-ClusterStateUpdateTest-seed#[7FC47D48B54585CC]-worker) [n:127.0.0.1:1661_solr    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.ClusterStateUpdateTest_7FC47D48B54585CC-001
   [junit4]   2> Jun 06, 2016 5:51:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene60, sim=ClassicSimilarity, locale=hi, timezone=Asia/Hebron
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=12,threads=1,free=182924104,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [CollectionsAPIDistributedZkTest, TestPivotHelperCode, BlockDirectoryTest, UniqFieldsUpdateProcessorFactoryTest, TestScoreJoinQPNoScore, FacetPivotSmallTest, FastVectorHighlighterTest, TestSolrDeletionPolicy2, HdfsThreadLeakTest, CSVRequestHandlerTest, TestSerializedLuceneMatchVersion, DocumentBuilderTest, SecurityConfHandlerTest, SuggestComponentTest, TestDynamicFieldResource, HdfsCollectionsAPIDistributedZkTest, TestEmbeddedSolrServerConstructors, TestChildDocTransformer, TestSortByMinMaxFunction, NumericFieldsTest, TestCodecSupport, SoftAutoCommitTest, HttpPartitionTest, TestCoreContainer, TestRestManager, TestFastLRUCache, DistribCursorPagingTest, TestQueryUtils, TestStressUserVersions, HighlighterMaxOffsetTest, TestQuerySenderListener, NotRequiredUniqueKeyTest, TestLegacyFieldCache, ShardRoutingCustomTest, MoreLikeThisHandlerTest, LoggingHandlerTest, CursorMarkTest, CloudMLTQParserTest, TestLMJelinekMercerSimilarityFactory, OpenExchangeRatesOrgProviderTest, DistributedQueueTest, TestAuthorizationFramework, TestCloudDeleteByQuery, SpellCheckComponentTest, DocValuesMissingTest, MinimalSchemaTest, SolrXmlInZkTest, ShardRoutingTest, BasicFunctionalityTest, CloneFieldUpdateProcessorFactoryTest, ChaosMonkeyNothingIsSafeTest, TestCloudSchemaless, TestFreeTextSuggestions, TestInfoStreamLogging, TestDocumentBuilder, TestQueryWrapperFilter, SortByFunctionTest, TestDFRSimilarityFactory, CollectionReloadTest, TestFiltering, TestSolrCloudWithKerberosAlt, TestMissingGroups, TestSubQueryTransformerDistrib, DistanceFunctionTest, SolrCmdDistributorTest, CdcrUpdateLogTest, RecoveryAfterSoftCommitTest, TermsComponentTest, TestUpdate, HdfsWriteToMultipleCollectionsTest, TestGeoJSONResponseWriter, DistributedExpandComponentTest, ConfigSetsAPITest, TestStressVersions, SolrInfoMBeanTest, DistributedQueryComponentCustomSortTest, HighlighterTest, DistributedTermsComponentTest, ZkSolrClientTest, TestSSLRandomization, TestDistributedGrouping, BitVectorTest, CoreAdminRequestStatusTest, TestTestInjection, TestBulkSchemaConcurrent, MultiTermTest, TestLRUStatsCache, TestJsonFacets, TestBM25SimilarityFactory, TestRawTransformer, TestIntervalFaceting, TimeZoneUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsRecoveryZkTest, TestManagedSchemaAPI, TestTolerantUpdateProcessorCloud, TestSearcherReuse, ClusterStateUpdateTest]
   [junit4] Completed [611/613 (9!)] on J1 in 322.28s, 1 test, 1 error <<< FAILURES!

[...truncated 26 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 5 errors, 5 failures, 1517 ignored (58 assumptions) [seed: 7FC47D48B54585CC]

Total time: 109 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (32bit/jdk-9-ea+121) - Build # 835 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/835/
Java: 32bit/jdk-9-ea+121 -client -XX:+UseG1GC

20 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'X val' for path 'x' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{"wt":"json"},   "context":{     "webapp":"/n_dxi",     "path":"/test1",     "httpMethod":"GET"},   "class":"org.apache.solr.core.BlobStoreTestRequestHandler",   "x":null},  from server:  null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'X val' for path 'x' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{"wt":"json"},
  "context":{
    "webapp":"/n_dxi",
    "path":"/test1",
    "httpMethod":"GET"},
  "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
  "x":null},  from server:  null
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:288D5F9496DF61E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:457)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:233)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.test

Error Message:
[[], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a1",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a1' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol1",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol1' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr1",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr1' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a2",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a2' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol2",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr2",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [   {     "replace-field":{       "name":"a3",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The field 'a3' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-dynamic-field":{       "name":"*_lol3",       "type":"text",       "stored":true,       "indexed":true},     "errorMessages":["The dynamic field '*_lol3' is not present in this schema, and so cannot be replaced.\n"]},   {     "replace-field-type":{       "name":"mystr3",       "class":"solr.TextField"},     "errorMessages":["The field type 'mystr3' is not present in this schema, and so cannot be replaced.\n"]}]], []]

Stack Trace:
java.lang.AssertionError: [[], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a1",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a1' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol1",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol1' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr1",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr1' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a2",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a2' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol2",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol2' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr2",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr2' is not present in this schema, and so cannot be replaced.\n"]}]], [["Error reading input String Can't find resource 'schema.xml' in classpath or '/configs/conf1', cwd=/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J1"], [
  {
    "replace-field":{
      "name":"a3",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The field 'a3' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-dynamic-field":{
      "name":"*_lol3",
      "type":"text",
      "stored":true,
      "indexed":true},
    "errorMessages":["The dynamic field '*_lol3' is not present in this schema, and so cannot be replaced.\n"]},
  {
    "replace-field-type":{
      "name":"mystr3",
      "class":"solr.TextField"},
    "errorMessages":["The field type 'mystr3' is not present in this schema, and so cannot be replaced.\n"]}]], []]
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.schema.TestBulkSchemaConcurrent.test(TestBulkSchemaConcurrent.java:115)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    3) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    4) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=8644, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    7) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    8) Thread[id=8673, name=OverseerCollectionConfigSetProcessor-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=8646, name=OverseerStateUpdate-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=8672, name=OverseerStateUpdate-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=8670, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   15) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   16) Thread[id=8647, name=OverseerCollectionConfigSetProcessor-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   3) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   4) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   5) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=8644, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   7) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   8) Thread[id=8673, name=OverseerCollectionConfigSetProcessor-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=8646, name=OverseerStateUpdate-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  11) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  12) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  13) Thread[id=8672, name=OverseerStateUpdate-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)
        at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  14) Thread[id=8670, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  15) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  16) Thread[id=8647, name=OverseerCollectionConfigSetProcessor-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    2) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    3) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    4) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    5) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    6) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    9) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8675, name=Thread-3264, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
   2) Thread[id=8677, name=zkCallback-1602-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   3) Thread[id=8643, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:34642), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   4) Thread[id=8669, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[DAC5F8AEBEB053BE]-SendThread(127.0.0.1:39497), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   5) Thread[id=8678, name=zkCallback-1602-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   6) Thread[id=8680, name=zkCallback-1602-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   7) Thread[id=8679, name=zkCallback-1602-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   8) Thread[id=8648, name=OverseerHdfsCoreFailoverThread-96021316510941187-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   9) Thread[id=8674, name=OverseerHdfsCoreFailoverThread-96021337503170563-127.0.0.1:7000_solr-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(java.base@9-ea/Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
  10) Thread[id=8649, name=Thread-3262, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at java.lang.Object.wait(java.base@9-ea/Native Method)
        at java.lang.Object.wait(java.base@9-ea/Object.java:516)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE]:0)


FAILED:  org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test

Error Message:
Error from server at http://127.0.0.1:36719/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:36719/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:135)
	at org.apache.solr.cloud.AsyncMigrateRouteKeyTest.test(AsyncMigrateRouteKeyTest.java:44)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
Error from server at http://127.0.0.1:42284: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore

Stack Trace:
java.lang.AssertionError: Error from server at http://127.0.0.1:42284: Error CREATEing SolrCore 'onenodecollectioncore': Unable to create core [onenodecollectioncore] Caused by: Timed out getting coreNodeName for onenodecollectioncore
	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:162)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:71)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:38795/ny/y/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /ny/y/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:38795/ny/y/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /ny/y/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:152)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:39709/_/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_/collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(java.base@9-ea/Thread.java:843) Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1  at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)  at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)  at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)  at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39709/_/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_/collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=org.apache.solr.common.SolrException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: Timed out getting coreNodeName for collection1
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
	at org.apache.solr.core.CoreContainer.create

[...truncated too long message...]

t failure: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1080)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:256)
   [junit4]    > 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:418)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:109)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]    > 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:518)
   [junit4]    > 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
   [junit4]    > 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
   [junit4]    > 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]    > 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]    > 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
   [junit4]    > 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]    > Caused by: org.apache.solr.common.SolrException: Timed out getting coreNodeName for collection1
   [junit4]    > 	at org.apache.solr.cloud.ZkController.waitForCoreNodeName(ZkController.java:1387)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.doGetShardIdAndNodeNameProcess(ZkController.java:1365)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.preRegister(ZkController.java:1456)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:809)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:468)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]    > 	... 1 more
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DAC5F8AEBEB053BE:5291C774104C3E46]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:775)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.index(BaseDistributedSearchTestCase.java:483)
   [junit4]    > 	at org.apache.solr.cloud.TestShortCircuitedRequests.test(TestShortCircuitedRequests.java:39)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 9608726 INFO  (SUITE-TestShortCircuitedRequests-seed#[DAC5F8AEBEB053BE]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestShortCircuitedRequests_DAC5F8AEBEB053BE-001
   [junit4]   2> Jun 05, 2016 11:58:43 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=1384, maxMBSortInHeap=7.003051398770211, sim=ClassicSimilarity, locale=ar-SD, timezone=America/Chihuahua
   [junit4]   2> NOTE: Linux 4.4.0-21-generic i386/Oracle Corporation 9-ea (32-bit)/cpus=12,threads=1,free=38539992,total=232783872
   [junit4]   2> NOTE: All tests run in this JVM: [AddBlockUpdateTest, ShowFileRequestHandlerTest, TestRequestForwarding, HardAutoCommitTest, TestDynamicLoading, TestSolrFieldCacheMBean, TestSchemaManager, TestCollationField, ChangedSchemaMergeTest, TestFieldCollectionResource, HdfsChaosMonkeyNothingIsSafeTest, TestScoreJoinQPScore, TestXIncludeConfig, TestStressRecovery, TestTrieFacet, TestHighFrequencyDictionaryFactory, AssignTest, TestComplexPhraseQParserPlugin, ExplicitHLLTest, TestExclusionRuleCollectionAccess, TestSuggestSpellingConverter, UUIDUpdateProcessorFallbackTest, TestDocTermOrds, TestSolrConfigHandlerConcurrent, TestTolerantSearch, TestCharFilters, CircularListTest, TestSlowCompositeReaderWrapper, RequestHandlersTest, TestManagedResource, HighlighterConfigTest, TestFieldCacheVsDocValues, DeleteInactiveReplicaTest, TestNumericTerms32, SuggesterTSTTest, TestExactSharedStatsCache, TestHdfsUpdateLog, TestFilteredDocIdSet, DebugComponentTest, XmlUpdateRequestHandlerTest, TestAddFieldRealTimeGet, TestRecoveryHdfs, TestPHPSerializedResponseWriter, TestWriterPerf, TestConfigReload, DirectUpdateHandlerOptimizeTest, SuggesterWFSTTest, TestCloudInspectUtil, SolrTestCaseJ4Test, RemoteQueryErrorTest, CollectionStateFormat2Test, QueryParsingTest, CoreAdminHandlerTest, TestJmxMonitoredMap, TestSolrQueryParser, TestStressCloudBlindAtomicUpdates, SimpleCollectionCreateDeleteTest, TestNonDefinedSimilarityFactory, RulesTest, BJQParserTest, TestNoOpRegenerator, DistributedVersionInfoTest, TestFieldTypeResource, HdfsTlogReplayBufferedWhileIndexingTest, RollingRestartTest, TestExtendedDismaxParser, TestReversedWildcardFilterFactory, AutoCommitTest, TestIndexingPerformance, SampleTest, SuggesterTest, BasicDistributedZk2Test, StressHdfsTest, TestSchemaSimilarityResource, TestInitParams, DistributedQueryComponentOptimizationTest, LukeRequestHandlerTest, TestManagedStopFilterFactory, UnloadDistributedZkTest, TestShortCircuitedRequests]
   [junit4] Completed [594/613 (17!)] on J2 in 604.12s, 1 test, 1 error <<< FAILURES!

[...truncated 108 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 13 errors, 5 failures, 1258 ignored (69 assumptions) [seed: DAC5F8AEBEB053BE]

Total time: 206 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS-EA] Lucene-Solr-6.x-Linux (64bit/jdk-9-ea+121) - Build # 834 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Linux/834/
Java: 64bit/jdk-9-ea+121 -XX:+UseCompressedOops -XX:+UseParallelGC

25 tests failed.
FAILED:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt6']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response> <lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt0</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt6']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int></lst><arr name="fields"><lst><str name="name">_root_</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">_version_</str><str name="type">long</str><bool name="indexed">true</bool><bool name="stored">true</bool></lst><lst><str name="name">constantField</str><str name="type">tdouble</str></lst><lst><str name="name">id</str><str name="type">string</str><bool name="multiValued">false</bool><bool name="indexed">true</bool><bool name="required">true</bool><bool name="stored">true</bool></lst><lst><str name="name">newTestFieldInt0</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt1</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt10</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt100</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt101</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt102</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt103</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt104</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt105</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt106</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt107</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt108</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt109</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt11</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt110</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt111</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt112</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt113</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt114</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt115</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt116</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt117</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt118</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt119</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt12</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt120</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt121</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt122</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt123</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt124</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt125</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt126</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt127</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt128</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt129</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt13</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt130</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt131</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt132</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt133</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt134</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt135</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt136</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt137</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt138</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt139</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt14</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt140</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt141</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt142</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt143</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt144</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt145</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt146</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt147</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt148</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt149</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt15</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt150</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt151</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt152</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt153</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt154</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt155</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt156</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt157</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt158</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt159</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt16</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt160</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt161</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt162</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt163</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt164</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt165</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt166</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt167</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt168</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt169</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt17</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt170</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt171</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt172</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt173</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt174</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt175</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt176</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt177</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt178</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt179</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt18</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt180</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt181</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt182</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt183</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt184</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt185</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt186</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt187</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt188</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt189</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt19</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt190</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt191</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt192</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt193</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt194</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt195</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt196</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt197</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt198</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt199</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt2</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt20</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt200</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt201</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt202</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt203</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt204</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt205</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt206</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt207</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt208</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt209</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt21</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt210</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt211</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt212</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt213</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt214</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt215</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt216</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt217</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt218</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt219</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt22</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt220</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt221</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt222</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt223</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt224</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt225</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt226</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt227</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt228</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt229</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt23</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt230</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt231</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt232</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt233</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt234</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt235</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt236</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt237</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt238</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt239</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt24</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt240</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt241</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt242</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt243</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt244</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt245</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt246</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt247</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt248</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt249</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt25</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt250</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt251</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt252</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt253</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt254</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt255</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt256</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt257</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt258</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt259</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt26</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt260</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt261</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt262</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt263</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt264</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt265</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt266</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt267</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt268</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt269</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt27</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt270</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt271</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt272</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt273</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt274</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt275</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt276</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt277</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt278</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt279</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt28</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt280</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt281</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt282</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt283</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt284</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt285</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt286</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt287</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt288</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt289</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt29</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt290</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt291</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt292</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt293</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt294</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt295</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt296</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt297</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt298</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt299</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt3</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt30</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt300</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt301</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt302</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt303</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt304</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt305</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt306</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt307</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt308</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt309</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt31</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt310</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt311</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt312</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt313</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt314</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt315</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt316</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt317</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt318</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt319</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt32</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt320</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt321</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt322</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt323</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt324</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt325</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt326</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt327</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt328</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt329</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt33</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt330</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt331</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt332</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt333</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt334</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt335</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt336</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt337</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt338</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt339</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt34</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt340</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt341</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt342</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt343</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt344</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt345</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt346</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt347</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt348</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt349</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt35</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt350</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt351</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt352</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt353</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt354</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt355</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt356</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt357</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt358</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt359</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt36</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt360</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt361</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt362</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt363</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt364</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt365</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt366</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt367</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt368</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt369</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt37</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt370</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt371</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt372</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt373</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt374</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt375</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt376</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt377</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt378</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt379</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt38</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt380</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt381</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt382</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt383</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt384</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt385</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt386</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt387</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt388</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt389</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt39</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt390</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt391</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt392</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt393</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt394</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt395</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt396</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt397</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt398</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt399</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt4</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt40</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt400</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt401</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt402</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt403</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt404</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt405</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt406</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt407</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt408</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt409</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt41</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt410</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt411</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt412</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt413</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt414</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt415</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt416</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt417</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt418</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt419</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt42</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt420</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt421</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt422</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt423</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt424</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt425</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt426</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt427</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt428</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt429</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt43</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt430</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt431</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt432</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt433</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt434</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt435</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt436</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt437</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt438</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt439</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt44</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt440</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt441</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt442</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt443</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt444</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt445</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt446</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt447</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt448</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt449</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt45</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt46</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt47</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt48</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt49</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt5</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt50</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt51</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt52</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt53</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt54</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt55</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt56</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt57</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt58</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt59</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt60</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt61</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt62</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt63</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt64</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt65</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt66</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt67</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt68</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt69</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt7</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt70</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt71</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt72</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt73</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt74</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt75</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt76</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt77</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt78</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt79</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt8</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt80</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt81</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt82</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt83</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt84</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt85</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt86</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt87</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt88</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt89</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt9</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt90</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt91</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt92</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt93</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt94</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt95</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt96</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt97</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt98</str><str name="type">tlong</str></lst><lst><str name="name">newTestFieldInt99</str><str name="type">tlong</str></lst></arr>
</response>

	at __randomizedtesting.SeedInfo.seed([56E77ED5C0190ED7:DEB3410F6EE5632F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:139)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@9-ea/Method.java:531)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionIntegrationTest

Error Message:
16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest:     1) Thread[id=6587, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=6614, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)         at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    3) Thread[id=6619, name=Thread-1882, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    4) Thread[id=6623, name=zkCallback-1401-thread-4, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    5) Thread[id=6613, name=TEST-LeaderElectionIntegrationTest.testLeaderElectionAfterClientTimeout-seed#[56E77ED5C0190ED7]-SendThread(127.0.0.1:41242), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=6622, name=zkCallback-1401-thread-3, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    7) Thread[id=6617, name=OverseerCollectionConfigSetProcessor-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)    8) Thread[id=6592, name=Thread-1880, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Object.wait(java.base@9-ea/Native Method)         at java.lang.Object.wait(java.base@9-ea/Object.java:516)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1253)    9) Thread[id=6616, name=OverseerStateUpdate-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:309)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:288)         at org.apache.solr.cloud.DistributedQueue.firstElement(DistributedQueue.java:357)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   10) Thread[id=6624, name=zkCallback-1401-thread-5, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   11) Thread[id=6618, name=OverseerHdfsCoreFailoverThread-96020038195347459-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   12) Thread[id=6621, name=zkCallback-1401-thread-2, state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(java.base@9-ea/LockSupport.java:230)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@9-ea/SynchronousQueue.java:461)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@9-ea/SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(java.base@9-ea/SynchronousQueue.java:937)         at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@9-ea/ThreadPoolExecutor.java:1082)         at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1143)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   13) Thread[id=6591, name=OverseerHdfsCoreFailoverThread-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   14) Thread[id=6589, name=OverseerStateUpdate-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:418)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)   15) Thread[id=6586, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-SendThread(127.0.0.1:38500), state=TIMED_WAITING, group=TGRP-LeaderElectionIntegrationTest]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   16) Thread[id=6590, name=OverseerCollectionConfigSetProcessor-96020017207312387-127.0.0.1:7000_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(java.base@9-ea/Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:328)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:167)         at java.lang.Thread.run(java.base@9-ea/Thread.java:843)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 16 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderElectionIntegrationTest: 
   1) Thread[id=6587, name=TEST-LeaderElectionIntegrationTest.testSimpleSliceLeaderElection-seed#[56E77ED5C0190ED7]-EventThread, state=WAITING, group=TGRP-LeaderElectionIntegrationTest]
        at jdk.internal.misc.Unsafe.park(java.base@9-ea/Native Method)
        at java.util.concurrent.locks.LockSupport.park(java.base@9-ea/LockSupport.java:190)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@9-ea/AbstractQueuedSynchronizer.java:2064)
        at java.util.concurrent.LinkedBlockingQueue.take(java.base@9-ea/LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thre

[...truncated too long message...]

n/96020058430242830-core_node2-n_0000000002
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:244)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:241)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:86)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.cancelElection(ElectionContext.java:143)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.cancelElection(ElectionContext.java:279)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:455)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(java.base@9-ea/Executors.java:514)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(java.base@9-ea/FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@9-ea/ThreadPoolExecutor.java:1158)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@9-ea/ThreadPoolExecutor.java:632)
   [junit4]   2> 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6e562c3b{HTTP/1.1,[http/1.1]}{127.0.0.1:43507}
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1684a0c{/qz_q/x,null,UNAVAILABLE}
   [junit4]   2> 10577119 INFO  (TEST-RecoveryZkTest.test-seed#[56E77ED5C0190ED7]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45966 45966
   [junit4]   2> 10577144 INFO  (Thread-3279) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45966 45966
   [junit4]   2> 10577144 WARN  (Thread-3279) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/collections/collection1/state.json
   [junit4]   2> 	3	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/96020058430242824-127.0.0.1:34056_qz_q%2Fx-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=56E77ED5C0190ED7 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-SX -Dtests.timezone=Africa/Abidjan -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  626s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([56E77ED5C0190ED7:DEB3410F6EE5632F]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:127)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@9-ea/Native Method)
   [junit4]    > 	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@9-ea/NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@9-ea/DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
   [junit4]    > 	at java.lang.Thread.run(java.base@9-ea/Thread.java:843)
   [junit4]   2> 10577145 INFO  (SUITE-RecoveryZkTest-seed#[56E77ED5C0190ED7]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_56E77ED5C0190ED7-001
   [junit4]   2> Jun 05, 2016 6:14:03 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, maxPointsInLeafNode=908, maxMBSortInHeap=7.330895656806706, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=en-SX, timezone=Africa/Abidjan
   [junit4]   2> NOTE: Linux 4.4.0-21-generic amd64/Oracle Corporation 9-ea (64-bit)/cpus=12,threads=1,free=96044600,total=455081984
   [junit4]   2> NOTE: All tests run in this JVM: [TestDistribDocBasedVersion, DeleteStatusTest, ConnectionReuseTest, IndexSchemaRuntimeFieldTest, InfoHandlerTest, TestCustomDocTransformer, TestImplicitCoreProperties, TestLMDirichletSimilarityFactory, TestSolrQueryParserResource, OverseerTaskQueueTest, PingRequestHandlerTest, TestGraphTermsQParserPlugin, AnalyticsQueryTest, PolyFieldTest, PKIAuthenticationIntegrationTest, TestConfig, ForceLeaderTest, UUIDFieldTest, TestDocBasedVersionConstraints, TestRawResponseWriter, TestRangeQuery, TestLazyCores, TestFoldingMultitermQuery, MultiThreadedOCPTest, PathHierarchyTokenizerFactoryTest, URLClassifyProcessorTest, TestSchemaVersionResource, SystemInfoHandlerTest, TestMiniSolrCloudClusterSSL, HdfsRecoverLeaseTest, TestCloudPivotFacet, BasicZkTest, IndexSchemaTest, RAMDirectoryFactoryTest, SynonymTokenizerTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, VersionInfoTest, SparseHLLTest, TestSQLHandler, DistributedSpellCheckComponentTest, QueryEqualityTest, DistanceUnitsTest, TestManagedSchema, TestDistributedSearch, TestCustomSort, DistributedSuggestComponentTest, TestCrossCoreJoin, OverseerCollectionConfigSetProcessorTest, CdcrVersionReplicationTest, TestValueSourceCache, TestComponentsName, TestCloudBackupRestore, BigEndianAscendingWordSerializerTest, EchoParamsTest, LeaderInitiatedRecoveryOnCommitTest, TestConfigSetsAPIExclusivity, TestLRUCache, SuggestComponentContextFilterQueryTest, FileBasedSpellCheckerTest, CreateCollectionCleanupTest, TestUseDocValuesAsStored, TestStressLucene, TestLuceneMatchVersion, TestOmitPositions, TestOverriddenPrefixQueryForCustomFieldType, TestLeaderElectionZkExpiry, SignatureUpdateProcessorFactoryTest, TestDocSet, TestCSVLoader, HdfsChaosMonkeySafeLeaderTest, ResourceLoaderTest, TestSolrConfigHandler, TestFaceting, TestDefaultStatsCache, TestMiniSolrCloudCluster, TestFastOutputStream, ConjunctionSolrSpellCheckerTest, TestExactStatsCache, TestClusterStateMutator, DistributedFacetPivotLongTailTest, TlogReplayBufferedWhileIndexingTest, TestFieldResource, TestManagedSynonymFilterFactory, TestSolr4Spatial, DocumentAnalysisRequestHandlerTest, HdfsSyncSliceTest, TestNamedUpdateProcessors, SSLMigrationTest, StandardRequestHandlerTest, TestAnalyzedSuggestions, TestRandomDVFaceting, QueryResultKeyTest, PrimUtilsTest, TestRemoteStreaming, DeleteLastCustomShardedReplicaTest, GraphQueryTest, AsyncCallRequestStatusResponseTest, TestClassicSimilarityFactory, BasicDistributedZkTest, UnloadDistributedZkTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest]
   [junit4] Completed [613/613 (22!)] on J0 in 626.76s, 1 test, 1 failure <<< FAILURES!

[...truncated 20 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:740: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:684: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/build.xml:59: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/build.xml:233: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/solr/common-build.xml:531: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:1427: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-6.x-Linux/lucene/common-build.xml:984: There were test failures: 613 suites (10 ignored), 2567 tests, 2 suite-level errors, 18 errors, 5 failures, 1199 ignored (71 assumptions) [seed: 56E77ED5C0190ED7]

Total time: 196 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any