You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2023/03/27 05:04:03 UTC

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-19) - Build # 11020 - Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/11020/
Java: 64bit/hotspot/jdk-19 -XX:-UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.
	at java.base@19/java.lang.Thread.getStackTrace(Thread.java:2550)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.tests.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:804)
	at app//org.apache.lucene.tests.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1458)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1323)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1305)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:612)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1124)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1053)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1659)
	at app//org.apache.solr.core.CoreContainer.lambda$load$12(CoreContainer.java:1008)
	at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:234)
	at java.base@19/java.util.concurrent.FutureTask.run(FutureTask.java:317)
	at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@7bdbd995 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7dfb72b2)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.tests.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.
	at java.base@19/java.lang.Thread.getStackTrace(Thread.java:2550)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.tests.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:804)
	at app//org.apache.lucene.tests.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1458)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1323)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1305)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:418)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:862)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1128)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1053)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1659)
	at app//org.apache.solr.core.CoreContainer.lambda$load$12(CoreContainer.java:1008)
	at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:234)
	at java.base@19/java.util.concurrent.FutureTask.run(FutureTask.java:317)
	at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(NRTCachingDirectory(ByteBuffersDirectory@f7f0226 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@439c427e; maxCacheMB=0.893671989440918 maxMergeSizeMB=0.2746744155883789))
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.tests.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1111, name=coreLoadExecutor-546-thread-2-processing-127.0.0.1:38211_, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.
	at java.base@19/java.lang.Thread.getStackTrace(Thread.java:2550)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.tests.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:804)
	at app//org.apache.lucene.tests.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1458)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1323)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1305)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:901)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1128)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1053)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1659)
	at app//org.apache.solr.core.CoreContainer.lambda$load$12(CoreContainer.java:1008)
	at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:234)
	at java.base@19/java.util.concurrent.FutureTask.run(FutureTask.java:317)
	at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@651c359c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@de2e49c)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.tests.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1148, name=recoveryExecutor-538-thread-1-processing-127.0.0.1:38211_ collection1_shard1_replica_n3 collection1 shard1 core_node4, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=1148, name=recoveryExecutor-538-thread-1-processing-127.0.0.1:38211_ collection1_shard1_replica_n3 collection1 shard1 core_node4, state=RUNNABLE, group=TGRP-BasicDistributedZkTest], registration stack trace below.
	at java.base@19/java.lang.Thread.getStackTrace(Thread.java:2550)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.tests.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:804)
	at app//org.apache.lucene.tests.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1458)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1323)
	at app//org.apache.lucene.tests.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1305)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:614)
	at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:424)
	at app//org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:483)
	at app//org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:247)
	at app//org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:703)
	at app//org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:335)
	at app//org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:311)
	at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:212)
	at java.base@19/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:577)
	at java.base@19/java.util.concurrent.FutureTask.run(FutureTask.java:317)
	at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base@19/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@2e34a379 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@24e5e808)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.tests.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   1) Thread[id=1148, name=recoveryExecutor-538-thread-1-processing-127.0.0.1:38211_ collection1_shard1_replica_n3 collection1 shard1 core_node4, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.Net.poll(Native Method)
        at java.base/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:186)
        at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:275)
        at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
        at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
        at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
        at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:602)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:240)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.getStream(IndexFetcher.java:1990)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.fetch(IndexFetcher.java:1795)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.fetchFile(IndexFetcher.java:1779)
        at org.apache.solr.handler.IndexFetcher.downloadIndexFiles(IndexFetcher.java:1200)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:687)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:424)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:483)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:247)
        at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:703)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:335)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:311)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:212)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:577)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=1102, name=h2sc-537-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:180)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:187)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:604)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:541)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:450)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:243)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:194)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=1143, name=searcherExecutor-569-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3744)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3689)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=1194, name=fsyncService-627-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3744)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3689)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   1) Thread[id=1148, name=recoveryExecutor-538-thread-1-processing-127.0.0.1:38211_ collection1_shard1_replica_n3 collection1 shard1 core_node4, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.Net.poll(Native Method)
        at java.base/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:186)
        at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:275)
        at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
        at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
        at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
        at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
        at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:602)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:240)
        at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.getStream(IndexFetcher.java:1990)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.fetch(IndexFetcher.java:1795)
        at org.apache.solr.handler.IndexFetcher$FileFetcher.fetchFile(IndexFetcher.java:1779)
        at org.apache.solr.handler.IndexFetcher.downloadIndexFiles(IndexFetcher.java:1200)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:687)
        at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:424)
        at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:483)
        at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:247)
        at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:703)
        at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:335)
        at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:311)
        at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:212)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:577)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=1102, name=h2sc-537-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:180)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:187)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:604)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:541)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:450)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:243)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:194)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:289)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=1143, name=searcherExecutor-569-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3744)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3689)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=1194, name=fsyncService-627-thread-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3744)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3689)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
        at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([F22FCB5FC1AED373]:0)