You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2021/04/14 10:18:28 UTC

[JENKINS] Lucene-Solr-8.8-Linux (64bit/jdk-15) - Build # 65 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.8-Linux/65/
Java: 64bit/jdk-15 -XX:+UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingCustomTest

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingCustomTest:     1) Thread[id=928, name=qtp2115787492-928, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    2) Thread[id=952, name=SolrRrdBackendFactory-573-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)    3) Thread[id=930, name=qtp2115787492-930, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    4) Thread[id=929, name=qtp2115787492-929, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    5) Thread[id=931, name=qtp2115787492-931, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    6) Thread[id=936, name=qtp2115787492-936-acceptor-1@7c1a69ec-ServerConnector@68084970{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:35601}, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.Net.accept(Native Method)         at java.base@15/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:305)         at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)         at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    7) Thread[id=974, name=qtp2115787492-974, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)    8) Thread[id=932, name=qtp2115787492-932, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    9) Thread[id=933, name=qtp2115787492-933, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   10) Thread[id=926, name=qtp2115787492-926, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   11) Thread[id=953, name=MetricsHistoryHandler-574-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   12) Thread[id=954, name=Thread-154, state=WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/java.lang.Object.wait(Native Method)         at java.base@15/java.lang.Object.wait(Object.java:321)         at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)   13) Thread[id=973, name=qtp2115787492-973, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)   14) Thread[id=935, name=qtp2115787492-935-acceptor-0@2241b64d-ServerConnector@68084970{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:35601}, state=WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)         at java.base@15/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)         at java.base@15/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)         at java.base@15/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:300)         at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)         at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   15) Thread[id=958, name=qtp2115787492-958, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)   16) Thread[id=957, name=Connector-Scheduler-68084970-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   17) Thread[id=927, name=qtp2115787492-927, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   18) Thread[id=934, name=Session-HouseKeeper-215be448-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   19) Thread[id=955, name=coreContainerWorkExecutor-556-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@15/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   20) Thread[id=956, name=qtp2115787492-956, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.ShardRoutingCustomTest: 
   1) Thread[id=928, name=qtp2115787492-928, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   2) Thread[id=952, name=SolrRrdBackendFactory-573-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   3) Thread[id=930, name=qtp2115787492-930, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   4) Thread[id=929, name=qtp2115787492-929, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   5) Thread[id=931, name=qtp2115787492-931, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   6) Thread[id=936, name=qtp2115787492-936-acceptor-1@7c1a69ec-ServerConnector@68084970{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:35601}, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.Net.accept(Native Method)
        at java.base@15/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:305)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   7) Thread[id=974, name=qtp2115787492-974, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   8) Thread[id=932, name=qtp2115787492-932, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   9) Thread[id=933, name=qtp2115787492-933, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  10) Thread[id=926, name=qtp2115787492-926, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  11) Thread[id=953, name=MetricsHistoryHandler-574-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  12) Thread[id=954, name=Thread-154, state=WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/java.lang.Object.wait(Native Method)
        at java.base@15/java.lang.Object.wait(Object.java:321)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
  13) Thread[id=973, name=qtp2115787492-973, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  14) Thread[id=935, name=qtp2115787492-935-acceptor-0@2241b64d-ServerConnector@68084970{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:35601}, state=WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
        at java.base@15/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
        at java.base@15/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
        at java.base@15/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:300)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  15) Thread[id=958, name=qtp2115787492-958, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  16) Thread[id=957, name=Connector-Scheduler-68084970-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  17) Thread[id=927, name=qtp2115787492-927, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  18) Thread[id=934, name=Session-HouseKeeper-215be448-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  19) Thread[id=955, name=coreContainerWorkExecutor-556-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@15/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  20) Thread[id=956, name=qtp2115787492-956, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
	at __randomizedtesting.SeedInfo.seed([73124535F96D4B76]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ShardRoutingCustomTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=928, name=qtp2115787492-928, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    2) Thread[id=952, name=SolrRrdBackendFactory-573-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)    3) Thread[id=930, name=qtp2115787492-930, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    4) Thread[id=929, name=qtp2115787492-929, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    5) Thread[id=931, name=qtp2115787492-931, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    6) Thread[id=936, name=qtp2115787492-936, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)    7) Thread[id=974, name=qtp2115787492-974, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)    8) Thread[id=932, name=qtp2115787492-932, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)    9) Thread[id=933, name=qtp2115787492-933, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   10) Thread[id=926, name=qtp2115787492-926, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   11) Thread[id=953, name=MetricsHistoryHandler-574-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   12) Thread[id=954, name=Thread-154, state=WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/java.lang.Object.wait(Native Method)         at java.base@15/java.lang.Object.wait(Object.java:321)         at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)   13) Thread[id=973, name=qtp2115787492-973, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)   14) Thread[id=935, name=qtp2115787492-935, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)   15) Thread[id=958, name=qtp2115787492-958, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)   16) Thread[id=957, name=Connector-Scheduler-68084970-1, state=WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)         at java.base@15/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   17) Thread[id=927, name=qtp2115787492-927, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]         at java.base@15/sun.nio.ch.EPoll.wait(Native Method)         at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)         at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)         at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)         at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)         at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)         at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)         at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)         at java.base@15/java.lang.Thread.run(Thread.java:832)   18) Thread[id=934, name=Session-HouseKeeper-215be448-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   19) Thread[id=955, name=coreContainerWorkExecutor-556-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@15/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)         at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)         at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)         at java.base@15/java.lang.Thread.run(Thread.java:832)   20) Thread[id=956, name=qtp2115787492-956, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]         at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)         at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)         at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)         at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)         at java.base@15/java.lang.Thread.run(Thread.java:832)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=928, name=qtp2115787492-928, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   2) Thread[id=952, name=SolrRrdBackendFactory-573-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   3) Thread[id=930, name=qtp2115787492-930, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   4) Thread[id=929, name=qtp2115787492-929, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   5) Thread[id=931, name=qtp2115787492-931, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   6) Thread[id=936, name=qtp2115787492-936, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   7) Thread[id=974, name=qtp2115787492-974, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   8) Thread[id=932, name=qtp2115787492-932, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
   9) Thread[id=933, name=qtp2115787492-933, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  10) Thread[id=926, name=qtp2115787492-926, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  11) Thread[id=953, name=MetricsHistoryHandler-574-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  12) Thread[id=954, name=Thread-154, state=WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/java.lang.Object.wait(Native Method)
        at java.base@15/java.lang.Object.wait(Object.java:321)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
  13) Thread[id=973, name=qtp2115787492-973, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  14) Thread[id=935, name=qtp2115787492-935, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  15) Thread[id=958, name=qtp2115787492-958, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  16) Thread[id=957, name=Connector-Scheduler-68084970-1, state=WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@15/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  17) Thread[id=927, name=qtp2115787492-927, state=RUNNABLE, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/sun.nio.ch.EPoll.wait(Native Method)
        at java.base@15/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
        at java.base@15/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@15/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:149)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:156)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:572)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:509)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$246/0x0000000800e209c8.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  18) Thread[id=934, name=Session-HouseKeeper-215be448-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@15/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  19) Thread[id=955, name=coreContainerWorkExecutor-556-thread-1, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@15/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@15/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
  20) Thread[id=956, name=qtp2115787492-956, state=TIMED_WAITING, group=TGRP-ShardRoutingCustomTest]
        at java.base@15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@15/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:842)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:892)
        at java.base@15/java.lang.Thread.run(Thread.java:832)
	at __randomizedtesting.SeedInfo.seed([73124535F96D4B76]:0)


FAILED:  org.apache.solr.cloud.ShardRoutingCustomTest.test

Error Message:
Error from server at https://127.0.0.1:35601: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35601: ADDREPLICA failed to create replica
	at __randomizedtesting.SeedInfo.seed([73124535F96D4B76:FB467AEF5791268E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
	at org.apache.solr.cloud.ShardRoutingCustomTest.doCustomSharding(ShardRoutingCustomTest.java:75)
	at org.apache.solr.cloud.ShardRoutingCustomTest.test(ShardRoutingCustomTest.java:46)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:832)

[JENKINS] Lucene-Solr-8.8-Linux (64bit/jdk1.8.0_201) - Build # 66 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.8-Linux/66/
Java: 64bit/jdk1.8.0_201 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:43751/solr]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:43751/solr]
	at __randomizedtesting.SeedInfo.seed([12A24CE0B4A20A:2A2F090E40147807]:0)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:346)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
	at org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection(PerReplicaStatesIntegrationTest.java:84)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:43751/solr
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
	... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
	... 53 more

[JENKINS] Lucene-Solr-8.8-Linux (64bit/jdk1.8.0_201) - Build # 66 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.8-Linux/66/
Java: 64bit/jdk1.8.0_201 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:43751/solr]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:43751/solr]
	at __randomizedtesting.SeedInfo.seed([12A24CE0B4A20A:2A2F090E40147807]:0)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:346)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
	at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
	at org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection(PerReplicaStatesIntegrationTest.java:84)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:43751/solr
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
	at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
	... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
	... 53 more